gecko/intl/chardet/tests/DetectCharset.cpp

400 lines
10 KiB
C++

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is mozilla.org code.
*
* The Initial Developer of the Original Code is
* Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
* Pierre Phaneuf <pp@ludusdesign.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either of the GNU General Public License Version 2 or later (the "GPL"),
* or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include "nsISupports.h"
#include "nsIComponentManager.h"
#include "nsICharsetDetector.h"
#include "nsICharsetDetectionObserver.h"
#include "nsComponentManagerUtils.h"
#include "nsServiceManagerUtils.h"
#include <stdio.h>
#include <stdlib.h>
#if defined(XP_WIN) || defined(XP_OS2)
#include <io.h>
#endif
#ifdef XP_UNIX
#include <unistd.h>
#endif
class nsStatis {
public:
nsStatis() { };
virtual ~nsStatis() { };
virtual PRBool HandleData(const char* aBuf, PRUint32 aLen) = 0;
virtual void DataEnd() = 0;
virtual void Report()=0;
};
class nsBaseStatis : public nsStatis {
public:
nsBaseStatis(unsigned char aL, unsigned char aH, float aR) ;
virtual ~nsBaseStatis() {};
virtual PRBool HandleData(const char* aBuf, PRUint32 aLen);
virtual void DataEnd() ;
virtual void Report();
protected:
unsigned char mLWordHi;
unsigned char mLWordLo;
private:
PRUint32 mNumOf2Bytes;
PRUint32 mNumOfLChar;
PRUint32 mNumOfLWord;
PRUint32 mLWordLength;
PRUint32 mLWordLen[10];
float mR;
PRBool mTailByte;
PRBool mLastLChar;
};
nsBaseStatis::nsBaseStatis(unsigned char aL, unsigned char aH, float aR)
{
mNumOf2Bytes = mNumOfLWord = mLWordLength = mNumOfLChar= 0;
mTailByte = mLastLChar = PR_FALSE;
for(PRUint32 i =0;i < 20; i++)
mLWordLen[i] = 0;
mLWordHi = aH;
mLWordLo = aL;
mR = aR;
}
PRBool nsBaseStatis::HandleData(const char* aBuf, PRUint32 aLen)
{
for(PRUint32 i=0; i < aLen; i++)
{
if(mTailByte)
mTailByte = PR_FALSE;
else
{
mTailByte = (0x80 == ( aBuf[i] & 0x80));
if(mTailByte)
{
mNumOf2Bytes++;
unsigned char a = (unsigned char) aBuf[i];
PRBool thisLChar = (( mLWordLo <= a) && (a <= mLWordHi));
if(thisLChar)
{
mNumOfLChar++;
mLWordLength++;
} else {
if(mLastLChar) {
mNumOfLWord++;
mLWordLen[ (mLWordLength > 10) ? 9 : (mLWordLength-1)]++;
mLWordLength =0 ;
}
}
mLastLChar = thisLChar;
} else {
if(mLastLChar) {
mNumOfLWord++;
mLWordLen[ (mLWordLength > 10) ? 9 : (mLWordLength-1)]++;
mLWordLength =0 ;
mLastLChar = PR_FALSE;
}
}
}
}
return PR_TRUE;
}
void nsBaseStatis::DataEnd()
{
if(mLastLChar) {
mNumOfLWord++;
mLWordLen[ (mLWordLength > 10) ? 9 : (mLWordLength-1)]++;
}
}
void nsBaseStatis::Report()
{
if(mNumOf2Bytes > 0)
{
/*
printf("LChar Ratio = %d : %d ( %5.3f)\n",
mNumOfLChar,
mNumOf2Bytes,
((float)mNumOfLChar / (float)mNumOf2Bytes) * 100);
*/
float rate = (float) mNumOfLChar / (float) mNumOf2Bytes;
float delta = (rate - mR) / mR;
delta *= delta * 1000;
#ifdef EXPERIMENT
printf("Exp = %f \n",delta);
#endif
}
/*
if(mNumOfLChar > 0)
printf("LWord Word = %d : %d (%5.3f)\n",
mNumOfLWord,
mNumOfLChar,
((float)mNumOfLWord / (float)mNumOfLChar) * 100);
if(mNumOfLWord > 0)
{
PRUint32 ac =0;
for(PRUint32 i=0;i<10;i++)
{
ac += mLWordLen[i];
printf("LWord Word Length[%d]= %d -> %5.3f%% %5.3f%%\n", i+1,
mLWordLen[i],
(((float)mLWordLen[i] / (float)mNumOfLWord) * 100),
(((float)ac / (float)mNumOfLWord) * 100));
}
}
*/
}
class nsSimpleStatis : public nsStatis {
public:
nsSimpleStatis(unsigned char aL, unsigned char aH, float aR,const char* aCharset) ;
virtual ~nsSimpleStatis() {};
virtual PRBool HandleData(const char* aBuf, PRUint32 aLen);
virtual void DataEnd() ;
virtual void Report();
protected:
unsigned char mLWordHi;
unsigned char mLWordLo;
private:
PRUint32 mNumOf2Bytes;
PRUint32 mNumOfLChar;
float mR;
const char* mCharset;
PRBool mTailByte;
};
nsSimpleStatis::nsSimpleStatis(unsigned char aL, unsigned char aH, float aR, const char* aCharset)
{
mNumOf2Bytes = mNumOfLChar= 0;
mTailByte = PR_FALSE;
mLWordHi = aH;
mLWordLo = aL;
mR = aR;
mCharset = aCharset;
}
PRBool nsSimpleStatis::HandleData(const char* aBuf, PRUint32 aLen)
{
for(PRUint32 i=0; i < aLen; i++)
{
if(mTailByte)
mTailByte = PR_FALSE;
else
{
mTailByte = (0x80 == ( aBuf[i] & 0x80));
if(mTailByte)
{
mNumOf2Bytes++;
unsigned char a = (unsigned char) aBuf[i];
PRBool thisLChar = (( mLWordLo <= a) && (a <= mLWordHi));
if(thisLChar)
mNumOfLChar++;
}
}
}
return PR_TRUE;
}
void nsSimpleStatis::DataEnd()
{
}
void nsSimpleStatis::Report()
{
if(mNumOf2Bytes > 0)
{
float rate = (float) mNumOfLChar / (float) mNumOf2Bytes;
float delta = (rate - mR) / mR;
delta = delta * delta * (float)100;
#ifdef EXPERIMENT
printf("Exp = %f \n",delta);
if(delta < 1.0)
printf("This is %s\n" ,mCharset);
#endif
}
}
//==========================================================
#define MAXBSIZE (1L << 13)
void usage() {
printf("Usage: DetectFile detector blocksize\n"
" detector: "
"ja_parallel_state_machine,"
"ko_parallel_state_machine,"
"zhcn_parallel_state_machine,"
"zhtw_parallel_state_machine,"
"zh_parallel_state_machine,"
"cjk_parallel_state_machine,"
"ruprob,"
"ukprob,"
"\n blocksize: 1 ~ %ld\n"
" Data are passed in from STDIN\n"
, MAXBSIZE);
}
class nsReporter : public nsICharsetDetectionObserver
{
NS_DECL_ISUPPORTS
public:
nsReporter() { };
virtual ~nsReporter() { };
NS_IMETHOD Notify(const char* aCharset, nsDetectionConfident aConf)
{
printf("RESULT CHARSET : %s\n", aCharset);
printf("RESULT Confident : %d\n", aConf);
return NS_OK;
};
};
NS_IMPL_ISUPPORTS1(nsReporter, nsICharsetDetectionObserver)
nsresult GetDetector(const char* key, nsICharsetDetector** det)
{
char buf[128];
strcpy(buf, NS_CHARSET_DETECTOR_CONTRACTID_BASE);
strcat(buf, key);
return CallCreateInstance(buf, det);
}
nsresult GetObserver(nsICharsetDetectionObserver** aRes)
{
*aRes = nsnull;
nsReporter* rep = new nsReporter();
if(rep) {
return rep->QueryInterface(NS_GET_IID(nsICharsetDetectionObserver) ,
(void**)aRes);
}
return NS_ERROR_OUT_OF_MEMORY;
}
int main(int argc, char** argv) {
char buf[MAXBSIZE];
PRUint32 bs;
if( 3 != argc )
{
usage();
printf("Need 2 arguments\n");
return(-1);
}
bs = atoi(argv[2]);
if((bs <1)||(bs>MAXBSIZE))
{
usage();
printf("blocksize out of range - %s\n", argv[2]);
return(-1);
}
nsresult rev = NS_OK;
nsICharsetDetector *det = nsnull;
rev = GetDetector(argv[1], &det);
if(NS_FAILED(rev) || (nsnull == det) ){
usage();
printf("Invalid Detector - %s\n", argv[1]);
printf("XPCOM ERROR CODE = %x\n", rev);
return(-1);
}
nsICharsetDetectionObserver *obs = nsnull;
rev = GetObserver(&obs);
if(NS_SUCCEEDED(rev)) {
rev = det->Init(obs);
NS_IF_RELEASE(obs);
if(NS_FAILED(rev))
{
printf("XPCOM ERROR CODE = %x\n", rev);
return(-1);
}
} else {
printf("XPCOM ERROR CODE = %x\n", rev);
return(-1);
}
size_t sz;
PRBool done = PR_FALSE;
nsSimpleStatis ks(0xb0,0xc8, (float)0.95952, "EUC-KR");
nsSimpleStatis js(0xa4,0xa5, (float)0.45006, "EUC-JP");
nsStatis* stat[2] = {&ks, &js};
PRUint32 i;
do
{
sz = read(0, buf, bs);
if(sz > 0) {
if(! done) {
#ifdef DEBUG_DetectCharset
printf("call DoIt %d\n",sz);
#endif
rev = det->DoIt( buf, sz, &done);
#ifdef DEBUG_DetectCharset
printf("DoIt return Done = %d\n",done);
#endif
if(NS_FAILED(rev))
{
printf("XPCOM ERROR CODE = %x\n", rev);
return(-1);
}
}
for(i=0;i<2;i++)
stat[i]->HandleData(buf, sz);
}
// } while((sz > 0) && (!done) );
} while(sz > 0);
if(!done)
{
#ifdef DEBUG_DetectCharset
printf("Done = %d\n",done);
printf("call Done %d\n",sz);
#endif
rev = det->Done();
if(NS_FAILED(rev))
{
printf("XPCOM ERROR CODE = %x\n", rev);
return(-1);
}
}
for(i=0;i<2;i++) {
stat[i]->DataEnd();
stat[i]->Report();
}
#ifdef DEBUG_DetectCharset
printf( "Done\n");
#endif
NS_IF_RELEASE(det);
#ifdef DEBUG_DetectCharset
printf( "Done 2\n");
#endif
return (0);
}