text_samples/htmltextsample/main.cpp

/* IBM Source Code */
/* (C) Copyright IBM Corp. 2009, 2012 */
/* Licensed Materials - Property of IBM */
/* US Government Users Restricted Rights - Use duplication or disclosure restricted by GSA Schedule Contract with IBM Corp. */
 
#include <string>
#include <vector>
#include <iostream>
#include <fstream>
 
#include "dca/dca_base.h"
#include "dca/dca_text_classification.h"
#include "dca/dca_callbacks.h"
 
using namespace dca;
 
const std::string S_ToolName = "htmltextsample";
const std::string S_ToolVersion = "1.2";
 
const std::string S_UsageString =
        "<redist-folder> <ticket> <product> <html-list-file> [locale]\n"
        "  redist-folder - the folder where the DCA is installed to\n"
        "  ticket - a valid ticket\n"
        "  product - the product associated with your ticket\n"
        "  html-list-file - file that includes the HTML files to classify\n"
        "  locale - optional locale used for printing out category names, default = en_US\n\n"
        ;
 
#ifdef WIN32
#       define DCA_BINDIR               "bin/Win32"
#else
#       define DCA_BINDIR               "bin/linux"
#endif
 
#define DCA_INITDIR             "init"
 
#define DCA_LOGDIR              "./logs"
 
static void SetupInitData( const std::string& redist_folder, InitData& initData )
{
        initData.binDir = redist_folder + DCA_BINDIR;
        initData.initDir = redist_folder + DCA_INITDIR;
        initData.logDir = DCA_LOGDIR;
}
 
static void SetupLicense( const std::string& ticket, const std::string& product,
        LicenseData& licenseData )
{
        licenseData.ticket      = ticket;
        licenseData.product     = product;
}
 
static void LoadHtmlFileList( const std::string& fileName, std::vector<std::string>& htmlList )
{
        std::ifstream fstream( fileName.c_str(), std::ios::in );
        
        if (!fstream.is_open()) {
                std::cout << "Opening HTML list file '" << fileName << "' failed!" << std::endl;
        }
 
        std::string line;
 
        while ( std::getline(fstream, line) )
        {
                if( !line.empty() && line[line.length()-1] == '\r')
                                line.erase( line.length() - 1 );
 
                if( !line.empty() )
                        htmlList.push_back( line );
        }
}
 
static std::string LoadBinaryFile( const std::string& fileName )
{
        std::ifstream fstream( fileName.c_str(), std::ios::in | std::ios::binary | std::ios::ate);
        if (!fstream.is_open()) return std::string();
 
        const size_t size = fstream.tellg();
        fstream.seekg( 0, std::ios::beg );
 
        char* buffer = new char [ size ];
        if (!buffer) return std::string();
 
        fstream.read( buffer, size );
        std::string file = std::string( buffer, size );
        delete [] buffer;
 
        return file;
}
 
void TestTextClassification( const std::string& htmlFile, const DcaInstance& myDca, 
        const HtmlTextClassifier& myTextClassifier, const dca::CategoriesInfo& myCategoriesInfo,
        const std::string& locale )
{
        std::cout << "\nStarting HTML Text Classification routine..." << std::endl;
 
        std::vector<std::string> myFiles;
        LoadHtmlFileList( htmlFile, myFiles );
 
        const dca::Categories myCategories = myCategoriesInfo.getCategories();
 
        // perform an HTML Text Classification with every of the HTML files...
        for( std::vector<std::string>::const_iterator F = myFiles.begin(),
                FEnd = myFiles.end(); F != FEnd; ++F ) {
 
                const std::string myFile = *F;
 
                std::cout << "Starting HTML Text Classification with file '" <<
                        myFile << "'" << std::endl;
 
                const std::string fileContents = LoadBinaryFile( myFile );
                if( fileContents.empty() ) {
                        std::cerr << "Got empty file '" << myFile << "', continuing with next file." << std::endl;
                        continue;
                }
 
                // create an HTML text object with given file contents
                const HtmlText myText = HtmlText::create( myDca, fileContents );
 
                // perform the classification
                TextClassificationResults myTextClassificationResults;          
                FunctionResult myFR = myTextClassifier.classify( myText, myTextClassificationResults );
 
                if( !myFR ) {
                        // we encounter an error... print detailed error code and continue with next file
                        std::cerr << "Got error from HTML Text Classification. Details: '" <<
                                myFR.getDescription() << "' (" << myFR.getReturnCode() <<
                                "). Continuing with next file" << std::endl;
                        continue;
                }
 
                if( !myTextClassificationResults.isCategorized() ) {
                        // no error, but probably not enough data to classify or HTML page does not match to
                        // classifier categories.
                        std::cout << "Results: No matching categories found." << std::endl;
                }
                else {
                        // we got at least one matching category.
                        const DCA_SIZE_TYPE NumOfTCs = myTextClassificationResults.size();
                        // iterate through all matched categories and print out the detailed scores and results
                        for( DCA_INDEX_TYPE i = 0; i < NumOfTCs; ++i ) {
                                const TextClassificationResult myTextResult = myTextClassificationResults[ i ];
                                const double score = myTextResult.score();
                                const DCA_CATEGORY_ID_TYPE catid = myTextResult.id();
                                const dca::Category myCategory = myCategories.byId(catid);
                                const std::string catname = myCategory.name( locale );
 
                                if( catid == CATEGORY_ID_TEXT_PORNOGRAPHY ) {
                                        std::cout << "Results: " << (i+1) << ".\t Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                                else if( catid == CATEGORY_ID_TEXT_WAREZ ) {
                                        std::cout << "Results: " << (i+1) << ".\t Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                                else if( catid == CATEGORY_ID_TEXT_GAMBLING ) {
                                        std::cout << "Results: " << (i+1) << ".\t Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                                else if( catid == CATEGORY_ID_TEXT_ANONYMOUS_PROXIES ) {
                                        std::cout << "Results: " << (i+1) << ".\t Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                                else if( catid == CATEGORY_ID_TEXT_ILLEGAL_DRUGS ) {
                                        std::cout << "Results: " << (i+1) << ".\t Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                                else if( catid == CATEGORY_ID_TEXT_WEAPONS ) {
                                        std::cout << "Results: " << (i+1) << ".\t Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                                else { // we got a new category not known at the time this sample has been created!
                                        std::cout << "Results: " << (i+1) << ".\t New Category: '" << catname << 
                                                "', Score: " << score << std::endl;
                                }
                        }
                }
 
        } // for
        std::cout << "Leaving HTML Text Classification routine." << std::endl;
}
 
static void PrintCategoriesInfo( const dca::CategoriesInfo& myCategoriesInfo, const std::string& locale )
{
        const dca::Categories myCategories = myCategoriesInfo.getCategories();
        const DCA_SIZE_TYPE numOfCategories = myCategories.size();
        std::cout << "\nAvailable are the following " << numOfCategories << " categories:\n";
 
        for( DCA_INDEX_TYPE i = 0; i < numOfCategories; ++i ) {
                        const dca::Category myCategory = myCategories[ i ];
                        std::cout << "(" << (i+1)
                                 << ") Category name=" << myCategory.name( locale )
                                         <<      ", id=" << myCategory.id()
                                         <<      ", groupId=" << myCategory.groupId() << std::endl;
        }
 
        {               
                const dca::Groups myGroups = myCategoriesInfo.getGroups();
                const DCA_SIZE_TYPE numOfGroups = myGroups.size();
                std::cout << "\nAvailable are the following " << numOfGroups << " groups:\n";
 
                for( DCA_INDEX_TYPE i = 0; i < numOfGroups; ++i ) {
                                const dca::Group myGroup = myGroups[ i ];
                                std::cout << "(" << (i+1)
                                                 << ") Group name=" << myGroup.name( locale )
                                                 << ", id=" << myGroup.id() << std::endl;
                }
        }
        
        {               
                const dca::Locales myLocales = myCategoriesInfo.getLocales();
                const DCA_SIZE_TYPE numOfLocales = myLocales.size();
                std::cout << "\nAvailable are the following " << numOfLocales << " locales:\n";
 
                for( DCA_INDEX_TYPE i = 0; i < numOfLocales; ++i ) {
                                const dca::Locale myLocale = myLocales[ i ];
                                std::cout << "(" << (i+1)
                                                 << ") Locale displayName='" << myLocale.displayName()
                                                 << "', languageId='" << myLocale.languageId() << "'"
                                                 << std::endl;
                }
        }
}
 
static void PrintToolHeader()
{
        std::cout << "IBM DCA Sample: " << S_ToolName << " (" << S_ToolVersion << ")" << std::endl;
}
 
static void PrintUsage( const char *name )
{
        std::cout << name << std::endl << "Usage:" << std::endl;
        std::cout << S_UsageString << std::endl;
}
 
int main( int argc, char *argv[] )
{
        PrintToolHeader();
 
        int rc = 0;
 
        try {
 
                if( argc < 5 ) {
                        PrintUsage( argv[0] );
                        return 5;
                }
 
                std::string redist_folder = argv[1];
                const std::string ticket = argv[2];
                const std::string product = argv[3];
                const std::string htmlFile = argv[4];
                std::string locale = "en_US";
 
                if( redist_folder.empty() || ticket.empty() ||
                        product.empty() || htmlFile.empty() ) {
                        PrintUsage( argv[0] );
                        return 5;
                }
 
                if (argc > 5)
                        locale = argv[5];
 
                // check for trailing fileslash - and add if necessary
                const char c = redist_folder[ redist_folder.length() - 1 ];
                if( c != '/' && c != '\\' )
                        redist_folder += "/";
 
                // init the 3rd party libraries
                InitCUrl();
                SetOpenSslCallbacks();
                
                {
                        // setup DCA directories
                        InitData myInitData;
                        SetupInitData( redist_folder, myInitData );
 
                        // instantiate the DCA API
                        DcaInstance myDca;
                        myDca = DcaInstance::create( myInitData );
 
                        // setup license data
                        LicenseData myLicenseData;
                        SetupLicense( ticket, product, myLicenseData );
                        const License myLicense = myDca.createLicense( myLicenseData );
 
                        if( !myLicense.isLicensed( TextClassification::ID ) ) {
                                std::cout << "DCA is not licensed!" << std::endl;
                        } 
                        // get internal DCA categories info related to Text Classification
                        const dca::CategoriesInfo myCategoriesInfo = myDca.getCategoriesInfo( DCA_CAT_INFO_TYPE_TEXT );
                        // print the available categories, groups and locales
                        PrintCategoriesInfo( myCategoriesInfo, locale );
 
                        // create the classification module and an HTML Text Classifier
                        TextClassification myTextClassification = TextClassification::create( myDca, myLicense );
                        const HtmlTextClassifier myTextClassifier = myTextClassification.createHtmlClassifier();
 
                        // call Text Classification routine
                        TestTextClassification( htmlFile, myDca, myTextClassifier, myCategoriesInfo, locale );
                }
        }
        catch( const ExDca& ex ) {
                std::cerr << "DCA Exception occured. Details: " <<  ex.getDescription() <<
                        " (" << ex.getReturnCode() << ")." << std::endl;
                rc = 10;
        }
        catch( const std::exception& s ) {
                std::cerr << "std::exception occured. Details: " << s.what() << "." << std::endl;
                rc = 10;
        }
        catch(...) {
                std::cerr << "Unknown exception caught." << std::endl;
                rc = 10;
        }
 
        // deinit the 3rd party libraries
        UnsetOpenSslCallbacks();
        DeinitCUrl();
 
        return rc;
}