url_samples/urldbsample_remote/main.cpp

/* IBM Source Code */
/* (C) Copyright IBM Corp. 2009, 2012 */
/* Licensed Materials - Property of IBM */
/* US Government Users Restricted Rights - Use duplication or disclosure restricted by GSA Schedule Contract with IBM Corp. */
 
#include <string>
#include <vector>
#include <iostream>
#include <fstream>
#include <cstdlib>
#include <ctime>
 
#ifdef WIN32
#       include <winsock2.h>
#endif
 
#include "dca/dca_base.h"
#include "dca/dca_url_classification.h"
#include "dca/dca_callbacks.h"
 
using namespace dca;
 
const std::string S_ToolName = "urldbsample_remote";
const std::string S_ToolVersion = "1.3";
 
std::string G_Locale = "en_US";         // Default locale
LogLevel G_LogLevel = LOG_Notice;       // Default log level
 
const std::string S_UsageString =
"<dca-redist-folder> <ticket> <product> <encryption-data> <encryption-key> "
"<url-list-file> [<locale>] [<log-level>]\n"
        "  dca-redist-folder - the folder where the DCA is installed to\n"
        "  ticket - a valid ticket\n"
        "  product - the product associated with your ticket\n"
        "  hex-encryption-data - the encryption data (as hex string) included in "
"your license\n"
        "  encryption-key - the encryption key included in your license\n"
        "  url-list-file - file that includes the URLs to classify\n"
        "  locale - optional locale for the categories names, default = en_US\n"
        "  log-level - optional log-level, default = 3 (LOG_Notice)\n\n"
        ;
 
#ifdef WIN32
#       define DCA_BINDIR               "bin/Win32"
#else
#       define DCA_BINDIR               "bin/linux"
#endif
 
#define DCA_INITDIR             "init"
 
#define DCA_LOGDIR              "./logs"
 
static void SetupInitData( const std::string& redist_folder, 
        InitData& initData )
{
        initData.binDir = redist_folder + DCA_BINDIR;
        initData.initDir = redist_folder + DCA_INITDIR;
        initData.logDir = DCA_LOGDIR;
}
 
static bool StartupLibraries()
{
#ifdef WIN32
        // Windows needs an extra socket-startup for this process to work 
        // correctly with e.g. IP(v6) input IP addresses
        WORD wVersionRequested = MAKEWORD( 2, 2 );
        WSADATA wsaData;
        int err = WSAStartup( wVersionRequested, &wsaData );
        if ( err != 0 ) {
                std::cout << "Error on WSAStartup (" << err << ") occured, aborting" << 
                        std::endl;
                        return false;
        }
#endif  
 
        // init the 3rd party libraries
        InitCUrl();
        SetOpenSslCallbacks();
        return true;
}
 
static void ShutdownLibraries()
{
        // deinit the 3rd party libraries
        UnsetOpenSslCallbacks();
        DeinitCUrl();
        
#ifdef WIN32    
        // Cleanup Windows sockets for this process
        WSACleanup();
#endif
}
 
 
static void SetupLicense( const std::string& ticket, const std::string& product,
        LicenseData& licenseData )
{
        licenseData.ticket      = ticket;
        licenseData.product     = product;
}
 
static bool SetupConnectionData( const std::string& encData, 
        const std::string& encKey, DbConnectionData& cData )
{
        const int iEncKey = atoi( encKey.c_str() );
 
        if( iEncKey <= 0 )
                return false;
 
        cData.useLocalDatabase = false;
        cData.dbType = DBT_Url;
        cData.remoteServerData.encryptionData = encData;
        cData.remoteServerData.encryptionKey = iEncKey;
 
        return true;
}
 
static void PrintResults( const CategoriesInfo& catinfos, 
        const UrlClassificationResults& cats )
{
        const DCA_SIZE_TYPE numOfCats = cats.size();
 
        if( numOfCats == 0 ) {
                std::cout << "    No categories found." << std::endl;
                return;
        }
 
        const Categories myCategories = catinfos.getCategories();
 
        for( DCA_INDEX_TYPE i = 0; i < numOfCats; ++i ) {
                const UrlClassificationResult result = cats[ i ];
                const Category myCategory = myCategories.byId( result );
                const std::string catname = myCategory.name( G_Locale );
 
                if( myCategory != NullCategory ) {
                        std::cout << "    " << (i+1) << ".\t Category '" << catname <<
                                "' (id=" << myCategory.id() <<
                                ", groupid=" << myCategory.groupId() <<
                                ")" << std::endl;
                }
        }
}
 
static void PrintToolHeader()
{
        std::cout << "IBM DCA Sample: " << S_ToolName << " (" << S_ToolVersion 
                << ")" << std::endl;
}
 
static void PrintUsage()
{
        std::cout << " usage:" << std::endl;
        std::cout << S_UsageString << std::endl;
}
 
static void PrintDbConnectionInfo( const DbConnection& aDbConnection )
{
        DatabaseInformation databaseInformation = 
                aDbConnection.getDatabaseInformation();
 
        std::cout << "URL Database Version: " << databaseInformation.versionString 
                << " as of " << databaseInformation.creationDateUTC << std::endl;
}
 
static void PrintLicenseInfo( const License& aLicense )
{
        const time_t expirationDate = aLicense.getExpirationDate();
        struct tm *expirationTime = localtime( &expirationDate );
 
        std::cout << "License Info:" << std::endl;
        std::cout << "  DCA is " << ( aLicense.isLicensed() ? "licensed." : 
                "not licensed." ) << std::endl;
        std::cout << "  MaxUsers:" << aLicense.getMaxUsers() << 
                std::endl;
        std::cout << "  MaxSessions:" << aLicense.getMaxSessions() << 
                std::endl;
        std::cout << "  Ticket:" << aLicense.getTicket() << 
                std::endl;
        std::cout << "  Session:" << aLicense.getSession() << 
                std::endl;
        std::cout << "  Last Message:" << aLicense.getLastMessage() << 
                std::endl;
        std::cout << "  Expiration Date:" << asctime( expirationTime ) << 
                std::endl;
}
 
static void LoadUrlFile( const std::string& fileName, 
        std::vector<std::string>& urlList )
{
        std::ifstream fstream( fileName.c_str(), std::ios::in );
        if (!fstream.is_open()) return;
 
        std::string line;
 
        while ( std::getline(fstream, line) )
        {
                if( !line.empty() && line[line.length()-1] == '\r')
                        line.erase( line.length() - 1 );
 
                if( !line.empty() )
                        urlList.push_back( line );
        }
}
 
void TestUrlClassification( const std::string& aUrlListFile, 
        const DcaInstance& myDca, const UrlDbClassifier& myUrlDbClassifier, 
        const CategoriesInfo& myCategoriesInfo )
{
        size_t urlsRequested = 0;
        size_t unknownUrls = 0;
        size_t uncategerizedUrls = 0;
        size_t categoriesFound = 0;
 
        std::cout << "Entering URL db classification routine..." <<  std::endl;
 
        UrlClassificationResults myUrlClassificationResults;
 
        std::vector<std::string> myUrlList;
        LoadUrlFile( aUrlListFile, myUrlList );
 
        for( std::vector<std::string>::const_iterator U = myUrlList.begin(),
                UEnd = myUrlList.end(); U != UEnd; ++U ) {
 
                const std::string& myUrlString = *U;
 
                std::cout << "  Starting URL db classification for URL '" <<
                        myUrlString << "'" << std::endl;
 
                // setup a URL for given string
                const Url myUrl = Url::create( myDca, myUrlString );
 
                ++urlsRequested;
 
                // start the classification
                FunctionResult myFR = 
                        myUrlDbClassifier.classify( myUrl, myUrlClassificationResults );
                if( !myFR ) {
                        // error occured.
                        std::cerr << "    Error from URL db classification. Details: " << 
                                myFR.getDescription() << " (" << myFR.getReturnCode() << 
                                "). Continuing with next URL." << std::endl;
                        continue;
                }
 
                if( myUrlClassificationResults.isUnknownUrl() ) {
                        // the URL is not known in the database
                        std::cout << "    Results: URL '" << myUrlString << 
                                "' is not known in the database. " << std::endl;
                        std::cout << "    Continuing with next URL." << std::endl;
                        ++unknownUrls;
                        continue;
                }
 
                if( !myUrlClassificationResults.isCategorized() ) {
                        // the URL is known but does not contains any category. This is 
                        // either a white-host or includes only categorized sub-folders.
                        std::cout << "    Results: URL '" << myUrlString << 
                                "' is not categorized (but known in database). " << std::endl;
                        std::cout << "    Continuing with next URL" << std::endl;
                        ++uncategerizedUrls;
                        continue;
                }
 
                // given URL is known and returned matched categories
                std::cout << "    Classification Results for URL '" << myUrlString << 
                        "'" << std::endl;
                PrintResults( myCategoriesInfo, myUrlClassificationResults );
                categoriesFound += myUrlClassificationResults.size();
 
        }
 
        std::cout << "  Total Results: " << std::endl;
        std::cout << "    URLs requested:\t\t" << urlsRequested << std::endl;
        std::cout << "    URLs unknown:\t\t" << unknownUrls << std::endl;
        std::cout << "    URLs not categorized:\t" << uncategerizedUrls << 
                std::endl;
        std::cout << "    URLs categorized:\t\t" <<
                ( urlsRequested  - unknownUrls - uncategerizedUrls ) << std::endl;
        std::cout << "    Categories found for URLs:\t" << categoriesFound << 
                std::endl;
 
        std::cout << "Leaving URL db classification routine." << std::endl;
}
 
std::string HexToString( const std::string& arg )
{
        if( (arg.length()%2) != 0 || arg.find( "0x" ) != 0 ) // not a hex string
                return arg;
 
        const std::string cmd( arg.substr( 2 ) );
        std::string result;
 
        for( std::string::size_type i = 0; i < cmd.length(); i += 2 ) {
                unsigned char hex[3] = { 0x00, 0x00, 0x00 };
                hex[0] = cmd[i];
                hex[1] = cmd[i+1];
                unsigned int x = 0;
                sscanf( (const char *)hex, "%02X", &x );
                result += (unsigned char)x;
        }
        return result;
}
 
int main( int argc, char *argv[] )
{
        PrintToolHeader();
 
        int rc = 5;
 
        try {
                if( argc < 7 ) {
                        PrintUsage();
                        return 5;
                }
 
                std::string myRedistFolder = argv[ 1 ];
                const std::string myTicket = argv[ 2 ];
                const std::string myProduct = argv[ 3 ];
 
                // encryption data may be entered in hex format 0xAB02FF....
                // or as a common string if only ASCII characters are used
                const std::string myEncData = HexToString( argv[ 4 ] );
 
                const std::string myEncKey = argv[ 5 ];
                const std::string myUrlList = argv[ 6 ];
 
                if (argc > 7)
                {
                        G_Locale = argv[7];
 
                        if (argc > 8)
                        {
                                G_LogLevel = static_cast<LogLevel>( atoi( argv[8] ) );
                        }
                }
 
                if( myRedistFolder.empty() || myTicket.empty() ||
                         myProduct.empty() || myUrlList.empty() ||
                         myEncData.empty() || myEncKey.empty() ) {
                        PrintUsage();
                        return 5;
                }
 
                // check for trailing fileslash - and add if necessary
                const char c = myRedistFolder[ myRedistFolder.length() - 1 ];
                if( c != '/' && c != '\\' )
                        myRedistFolder += "/";
 
                // Initialize socket on Windows and 3rd party libraries
                if( !StartupLibraries() ) 
                        return 5;
 
                {
                        // setup DCA directories
                        InitData myInitData;
                        SetupInitData( myRedistFolder, myInitData );
 
                        // instantiate DCA API
                        DcaInstance myDca;
                        myDca = DcaInstance::create( myInitData );
 
                        // setup license data
                        LicenseData myLicenseData;
                        SetupLicense( myTicket, myProduct, myLicenseData );
                        const License myLicense = myDca.createLicense( myLicenseData, 
                                ProxySettings(), G_LogLevel );
 
                        PrintLicenseInfo( myLicense );
                        
                        if( myLicense.isLicensed( UrlClassification::ID ) ) {
                                // setup a signature database connection
                                DbConnectionData myDbConnectionData;
                                if ( !SetupConnectionData( myEncData, myEncKey, myDbConnectionData ) ) {
                                        std::cout << "Could not setup remote connection because of wrong encryption data!" << std::endl;
                                        rc = 5;
                                }
                                else {
                                        const DbConnection myDbConnection = 
                                                myDca.createDbConnection( myLicense, myDbConnectionData, 
                                                        ProxySettings(), G_LogLevel );
                                        PrintDbConnectionInfo( myDbConnection );
 
                                        // initialize the URL classification module and create a URL 
                                        // db classifier
                                        // (statistics and unknown url upload disabled)
                                        UrlClassification myUrlClassification = 
                                                UrlClassification::create( myDca, myLicense );
                                        myUrlClassification.setLogLevel( G_LogLevel );
 
                                        UrlDbClassifierOptions myUrlDbClassifierOptions;
                                        myUrlDbClassifierOptions.enable_EmbeddedUrlDetection = true;
                                        myUrlDbClassifierOptions.detect_EmbeddedUrlsInUrlPath = true;
                                        myUrlDbClassifierOptions.enable_Feedback = false;
 
                                        UrlDbClassifier myUrlDbClassifier = 
                                                myUrlClassification.createDbClassifier( myDbConnection, myUrlDbClassifierOptions );
                                        myUrlDbClassifier.setLogLevel( G_LogLevel );
 
                                        // create a categories info for printing out the category names
                                        // together with the results per URL
                                        const CategoriesInfo myCategoriesInfo = 
                                                myDca.getCategoriesInfo( DCA_CAT_INFO_TYPE_URL );
 
                                        // call URL Classification routine
                                        TestUrlClassification( myUrlList, myDca, myUrlDbClassifier, 
                                                myCategoriesInfo );
 
                                        rc = 0;
                                }
                        }
                }
        }
        catch( const ExDca& ex ) {
                std::cerr << "DCA Exception occured. Details: " <<  ex.getDescription() 
                        << " (" << ex.getReturnCode() << ")." << std::endl;
                rc = 10;
        }
        catch( const std::exception& s ) {
                std::cerr << "std::exception occured. Details: " << s.what() << "." << 
                        std::endl;
                rc = 10;
        }
        catch(...) {
                std::cerr << "Unknown exception caught." << std::endl;
                rc = 10;
        }
 
        // deinit the 3rd party libraries
        ShutdownLibraries();
 
        return rc;
}