mbconvtest.cpp 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356
  1. ///////////////////////////////////////////////////////////////////////////////
  2. // Name: tests/mbconv/main.cpp
  3. // Purpose: wxMBConv unit test
  4. // Author: Vadim Zeitlin, Mike Wetherell, Vince Harron
  5. // Created: 14.02.04
  6. // Copyright: (c) 2003 TT-Solutions, (c) 2005 Mike Wetherell, Vince Harron
  7. ///////////////////////////////////////////////////////////////////////////////
  8. // ----------------------------------------------------------------------------
  9. // headers
  10. // ----------------------------------------------------------------------------
  11. #include "testprec.h"
  12. #ifdef __BORLANDC__
  13. #pragma hdrstop
  14. #endif
  15. #ifndef WX_PRECOMP
  16. #include "wx/wx.h"
  17. #endif // WX_PRECOMP
  18. #include "wx/strconv.h"
  19. #include "wx/string.h"
  20. #include "wx/txtstrm.h"
  21. #include "wx/mstream.h"
  22. #if defined wxHAVE_TCHAR_SUPPORT && !defined HAVE_WCHAR_H
  23. #define HAVE_WCHAR_H
  24. #endif
  25. // ----------------------------------------------------------------------------
  26. // Some wide character constants. "\uXXXX" escapes aren't supported by old
  27. // compilers such as VC++ 5 and g++ 2.95.
  28. // ----------------------------------------------------------------------------
  29. wchar_t u41[] = { 0x41, 0 };
  30. wchar_t u7f[] = { 0x7f, 0 };
  31. wchar_t u80[] = { 0x80, 0 };
  32. wchar_t u391[] = { 0x391, 0 };
  33. wchar_t u7ff[] = { 0x7ff, 0 };
  34. wchar_t u800[] = { 0x800, 0 };
  35. wchar_t u2620[] = { 0x2620, 0 };
  36. wchar_t ufffd[] = { 0xfffd, 0 };
  37. #if SIZEOF_WCHAR_T == 4
  38. wchar_t u10000[] = { 0x10000, 0 };
  39. wchar_t u1000a5[] = { 0x1000a5, 0 };
  40. wchar_t u10fffd[] = { 0x10fffd, 0 };
  41. #else
  42. wchar_t u10000[] = { 0xd800, 0xdc00, 0 };
  43. wchar_t u1000a5[] = { 0xdbc0, 0xdca5, 0 };
  44. wchar_t u10fffd[] = { 0xdbff, 0xdffd, 0 };
  45. #endif
  46. // ----------------------------------------------------------------------------
  47. // test class
  48. // ----------------------------------------------------------------------------
  49. class MBConvTestCase : public CppUnit::TestCase
  50. {
  51. public:
  52. MBConvTestCase() { }
  53. private:
  54. CPPUNIT_TEST_SUITE( MBConvTestCase );
  55. CPPUNIT_TEST( UTF32LETests );
  56. CPPUNIT_TEST( UTF32BETests );
  57. CPPUNIT_TEST( WC2CP1250 );
  58. CPPUNIT_TEST( UTF7Tests );
  59. CPPUNIT_TEST( UTF8Tests );
  60. CPPUNIT_TEST( UTF16LETests );
  61. CPPUNIT_TEST( UTF16BETests );
  62. CPPUNIT_TEST( CP932Tests );
  63. CPPUNIT_TEST( CP1252Tests ); // depends on UTF8 Decoder functioning correctly
  64. CPPUNIT_TEST( LibcTests );
  65. CPPUNIT_TEST( IconvTests );
  66. CPPUNIT_TEST( Latin1Tests );
  67. CPPUNIT_TEST( FontmapTests );
  68. CPPUNIT_TEST( BufSize );
  69. CPPUNIT_TEST( FromWCharTests );
  70. #ifdef HAVE_WCHAR_H
  71. CPPUNIT_TEST( UTF8_41 );
  72. CPPUNIT_TEST( UTF8_7f );
  73. CPPUNIT_TEST( UTF8_80 );
  74. CPPUNIT_TEST( UTF8_c2_7f );
  75. CPPUNIT_TEST( UTF8_c2_80 );
  76. CPPUNIT_TEST( UTF8_ce_91 );
  77. CPPUNIT_TEST( UTF8_df_bf );
  78. CPPUNIT_TEST( UTF8_df_c0 );
  79. CPPUNIT_TEST( UTF8_e0_a0_7f );
  80. CPPUNIT_TEST( UTF8_e0_a0_80 );
  81. CPPUNIT_TEST( UTF8_e2_98_a0 );
  82. CPPUNIT_TEST( UTF8_ef_bf_bd );
  83. CPPUNIT_TEST( UTF8_ef_bf_c0 );
  84. CPPUNIT_TEST( UTF8_f0_90_80_7f );
  85. CPPUNIT_TEST( UTF8_f0_90_80_80 );
  86. CPPUNIT_TEST( UTF8_f4_8f_bf_bd );
  87. CPPUNIT_TEST( UTF8PUA_f4_80_82_a5 );
  88. CPPUNIT_TEST( UTF8Octal_backslash245 );
  89. #endif // HAVE_WCHAR_H
  90. CPPUNIT_TEST_SUITE_END();
  91. void WC2CP1250();
  92. void UTF7Tests();
  93. void UTF8Tests();
  94. void UTF16LETests();
  95. void UTF16BETests();
  96. void UTF32LETests();
  97. void UTF32BETests();
  98. void CP932Tests();
  99. void CP1252Tests();
  100. void LibcTests();
  101. void FontmapTests();
  102. void BufSize();
  103. void FromWCharTests();
  104. void IconvTests();
  105. void Latin1Tests();
  106. // verifies that the specified multibyte sequence decodes to the specified wchar_t sequence
  107. void TestDecoder(
  108. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  109. size_t wideChars, // the number of wide characters at wideBuffer
  110. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  111. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  112. wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence
  113. int sizeofNull // number of bytes occupied by terminating null in this encoding
  114. );
  115. // verifies that the specified wchar_t sequence encodes to the specified multibyte sequence
  116. void TestEncoder(
  117. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  118. size_t wideChars, // the number of wide characters at wideBuffer
  119. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  120. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  121. wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence
  122. int sizeofNull // number of bytes occupied by terminating null in this encoding
  123. );
  124. #if wxUSE_UNICODE && wxUSE_STREAMS
  125. // use wxTextInputStream to exercise wxMBConv interface
  126. // (this reveals some bugs in certain wxMBConv subclasses)
  127. void TestStreamDecoder(
  128. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  129. size_t wideChars, // the number of wide characters at wideBuffer
  130. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  131. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  132. wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence
  133. );
  134. // use wxTextOutputStream to exercise wxMBConv interface
  135. // (this reveals some bugs in certain wxMBConv subclasses)
  136. void TestStreamEncoder(
  137. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  138. size_t wideChars, // the number of wide characters at wideBuffer
  139. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  140. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  141. wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence
  142. );
  143. #endif
  144. // tests the encoding and decoding capability of an wxMBConv object
  145. //
  146. // decodes the utf-8 bytes into wide characters
  147. // encodes the wide characters to compare against input multiBuffer
  148. // decodes the multiBuffer to compare against wide characters
  149. // decodes the multiBuffer into wide characters
  150. void TestCoder(
  151. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  152. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  153. const char* utf8Buffer, // the same character sequence as multiBuffer, encoded as UTF-8
  154. size_t utf8Bytes, // the byte length of the UTF-8 encoded character sequence
  155. wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence
  156. int sizeofNull // the number of bytes occupied by a terminating null in the converter's encoding
  157. );
  158. #ifdef HAVE_WCHAR_H
  159. // UTF-8 tests. Test the first, last and one in the middle for sequences
  160. // of each length
  161. void UTF8_41() { UTF8("\x41", u41); }
  162. void UTF8_7f() { UTF8("\x7f", u7f); }
  163. void UTF8_80() { UTF8("\x80", NULL); }
  164. void UTF8_c2_7f() { UTF8("\xc2\x7f", NULL); }
  165. void UTF8_c2_80() { UTF8("\xc2\x80", u80); }
  166. void UTF8_ce_91() { UTF8("\xce\x91", u391); }
  167. void UTF8_df_bf() { UTF8("\xdf\xbf", u7ff); }
  168. void UTF8_df_c0() { UTF8("\xdf\xc0", NULL); }
  169. void UTF8_e0_a0_7f() { UTF8("\xe0\xa0\x7f", NULL); }
  170. void UTF8_e0_a0_80() { UTF8("\xe0\xa0\x80", u800); }
  171. void UTF8_e2_98_a0() { UTF8("\xe2\x98\xa0", u2620); }
  172. void UTF8_ef_bf_bd() { UTF8("\xef\xbf\xbd", ufffd); }
  173. void UTF8_ef_bf_c0() { UTF8("\xef\xbf\xc0", NULL); }
  174. void UTF8_f0_90_80_7f() { UTF8("\xf0\x90\x80\x7f", NULL); }
  175. void UTF8_f0_90_80_80() { UTF8("\xf0\x90\x80\x80", u10000); }
  176. void UTF8_f4_8f_bf_bd() { UTF8("\xf4\x8f\xbf\xbd", u10fffd); }
  177. // test 'escaping the escape characters' for the two escaping schemes
  178. void UTF8PUA_f4_80_82_a5() { UTF8PUA("\xf4\x80\x82\xa5", u1000a5); }
  179. void UTF8Octal_backslash245() { UTF8Octal("\\245", L"\\245"); }
  180. // implementation for the utf-8 tests (see comments below)
  181. void UTF8(const char *charSequence, const wchar_t *wideSequence);
  182. void UTF8PUA(const char *charSequence, const wchar_t *wideSequence);
  183. void UTF8Octal(const char *charSequence, const wchar_t *wideSequence);
  184. void UTF8(const char *charSequence, const wchar_t *wideSequence, int option);
  185. #endif // HAVE_WCHAR_H
  186. DECLARE_NO_COPY_CLASS(MBConvTestCase)
  187. };
  188. // register in the unnamed registry so that these tests are run by default
  189. CPPUNIT_TEST_SUITE_REGISTRATION( MBConvTestCase );
  190. // also include in its own registry so that these tests can be run alone
  191. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( MBConvTestCase, "MBConvTestCase" );
  192. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( MBConvTestCase, "MBConv" );
  193. void MBConvTestCase::WC2CP1250()
  194. {
  195. static const struct Data
  196. {
  197. const wchar_t *wc;
  198. const char *cp1250;
  199. } data[] =
  200. {
  201. { L"hello", "hello" }, // test that it works in simplest case
  202. { L"\xBD of \xBD is \xBC", NULL }, // this should fail as cp1250 doesn't have 1/2
  203. };
  204. wxCSConv cs1250(wxFONTENCODING_CP1250);
  205. for ( size_t n = 0; n < WXSIZEOF(data); n++ )
  206. {
  207. const Data& d = data[n];
  208. if (d.cp1250)
  209. {
  210. CPPUNIT_ASSERT( strcmp(cs1250.cWC2MB(d.wc), d.cp1250) == 0 );
  211. }
  212. else
  213. {
  214. CPPUNIT_ASSERT( (const char*)cs1250.cWC2MB(d.wc) == NULL );
  215. }
  216. }
  217. }
  218. // Print an unsigned character array as a C unsigned character array.
  219. // NB: Please don't remove this function even though it's not used anywhere,
  220. // it's very useful when debugging a failed test.
  221. wxString CByteArrayFormat( const void* data, size_t len, const wxChar* name )
  222. {
  223. const unsigned char* bytes = (unsigned char*)data;
  224. wxString result;
  225. result.Printf( wxT("static const unsigned char %s[%i] = \n{"), name, (int)len );
  226. for ( size_t i = 0; i < len; i++ )
  227. {
  228. if ( i != 0 )
  229. {
  230. result.append( wxT(",") );
  231. }
  232. if ((i%16)==0)
  233. {
  234. result.append( wxT("\n ") );
  235. }
  236. wxString byte = wxString::Format( wxT("0x%02x"), bytes[i] );
  237. result.append(byte);
  238. }
  239. result.append( wxT("\n};\n") );
  240. return result;
  241. }
  242. // The following bytes represent the same string, containing Japanese and English
  243. // characters, encoded in several different formats.
  244. // encoded by iconv
  245. static const unsigned char welcome_utf7_iconv[84] =
  246. {
  247. 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63,
  248. 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65,
  249. 0x2e,0x20,0x20,0x2b,0x4d,0x46,0x6b,0x77,0x55,0x49,0x74,0x6d,0x57,0x39,0x38,0x77,
  250. 0x61,0x35,0x62,0x37,0x69,0x6e,0x45,0x77,0x6b,0x6a,0x42,0x5a,0x4d,0x49,0x73,0x77,
  251. 0x65,0x7a,0x42,0x47,0x4d,0x45,0x77,0x77,0x52,0x44,0x42,0x45,0x4d,0x47,0x63,0x77,
  252. 0x57,0x54,0x41,0x43
  253. };
  254. // encoded by wxWindows (iconv can decode this successfully)
  255. static const unsigned char welcome_utf7_wx[109] =
  256. {
  257. 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x2b,0x41,0x43,0x41,0x2d,0x74,0x6f,0x2b,0x41,
  258. 0x43,0x41,0x2d,0x6f,0x75,0x72,0x2b,0x41,0x43,0x41,0x2d,0x63,0x79,0x62,0x65,0x72,
  259. 0x2b,0x41,0x43,0x41,0x2d,0x73,0x70,0x61,0x63,0x65,0x2b,0x41,0x43,0x41,0x2d,0x66,
  260. 0x6f,0x72,0x63,0x65,0x2e,0x2b,0x41,0x43,0x41,0x41,0x49,0x44,0x42,0x5a,0x4d,0x46,
  261. 0x43,0x4c,0x5a,0x6c,0x76,0x66,0x4d,0x47,0x75,0x57,0x2b,0x34,0x70,0x78,0x4d,0x4a,
  262. 0x49,0x77,0x57,0x54,0x43,0x4c,0x4d,0x48,0x73,0x77,0x52,0x6a,0x42,0x4d,0x4d,0x45,
  263. 0x51,0x77,0x52,0x44,0x42,0x6e,0x4d,0x46,0x6b,0x77,0x41,0x67,0x2d
  264. };
  265. // encoded by iconv
  266. static const unsigned char welcome_utf8[89] =
  267. {
  268. 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63,
  269. 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65,
  270. 0x2e,0x20,0x20,0xe3,0x81,0x99,0xe3,0x81,0x90,0xe8,0xad,0xa6,0xe5,0xaf,0x9f,0xe3,
  271. 0x81,0xab,0xe9,0x9b,0xbb,0xe8,0xa9,0xb1,0xe3,0x82,0x92,0xe3,0x81,0x99,0xe3,0x82,
  272. 0x8b,0xe3,0x81,0xbb,0xe3,0x81,0x86,0xe3,0x81,0x8c,0xe3,0x81,0x84,0xe3,0x81,0x84,
  273. 0xe3,0x81,0xa7,0xe3,0x81,0x99,0xe3,0x80,0x82
  274. };
  275. // encoded by iconv
  276. static const unsigned char welcome_utf16le[106] =
  277. {
  278. 0x57,0x00,0x65,0x00,0x6c,0x00,0x63,0x00,0x6f,0x00,0x6d,0x00,0x65,0x00,0x20,0x00,
  279. 0x74,0x00,0x6f,0x00,0x20,0x00,0x6f,0x00,0x75,0x00,0x72,0x00,0x20,0x00,0x63,0x00,
  280. 0x79,0x00,0x62,0x00,0x65,0x00,0x72,0x00,0x20,0x00,0x73,0x00,0x70,0x00,0x61,0x00,
  281. 0x63,0x00,0x65,0x00,0x20,0x00,0x66,0x00,0x6f,0x00,0x72,0x00,0x63,0x00,0x65,0x00,
  282. 0x2e,0x00,0x20,0x00,0x20,0x00,0x59,0x30,0x50,0x30,0x66,0x8b,0xdf,0x5b,0x6b,0x30,
  283. 0xfb,0x96,0x71,0x8a,0x92,0x30,0x59,0x30,0x8b,0x30,0x7b,0x30,0x46,0x30,0x4c,0x30,
  284. 0x44,0x30,0x44,0x30,0x67,0x30,0x59,0x30,0x02,0x30
  285. };
  286. // encoded by iconv
  287. static const unsigned char welcome_utf16be[106] =
  288. {
  289. 0x00,0x57,0x00,0x65,0x00,0x6c,0x00,0x63,0x00,0x6f,0x00,0x6d,0x00,0x65,0x00,0x20,
  290. 0x00,0x74,0x00,0x6f,0x00,0x20,0x00,0x6f,0x00,0x75,0x00,0x72,0x00,0x20,0x00,0x63,
  291. 0x00,0x79,0x00,0x62,0x00,0x65,0x00,0x72,0x00,0x20,0x00,0x73,0x00,0x70,0x00,0x61,
  292. 0x00,0x63,0x00,0x65,0x00,0x20,0x00,0x66,0x00,0x6f,0x00,0x72,0x00,0x63,0x00,0x65,
  293. 0x00,0x2e,0x00,0x20,0x00,0x20,0x30,0x59,0x30,0x50,0x8b,0x66,0x5b,0xdf,0x30,0x6b,
  294. 0x96,0xfb,0x8a,0x71,0x30,0x92,0x30,0x59,0x30,0x8b,0x30,0x7b,0x30,0x46,0x30,0x4c,
  295. 0x30,0x44,0x30,0x44,0x30,0x67,0x30,0x59,0x30,0x02
  296. };
  297. // encoded by iconv
  298. static const unsigned char welcome_utf32le[212] =
  299. {
  300. 0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x63,0x00,0x00,0x00,
  301. 0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00,
  302. 0x74,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,
  303. 0x75,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x63,0x00,0x00,0x00,
  304. 0x79,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x72,0x00,0x00,0x00,
  305. 0x20,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x61,0x00,0x00,0x00,
  306. 0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x66,0x00,0x00,0x00,
  307. 0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00,
  308. 0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x59,0x30,0x00,0x00,
  309. 0x50,0x30,0x00,0x00,0x66,0x8b,0x00,0x00,0xdf,0x5b,0x00,0x00,0x6b,0x30,0x00,0x00,
  310. 0xfb,0x96,0x00,0x00,0x71,0x8a,0x00,0x00,0x92,0x30,0x00,0x00,0x59,0x30,0x00,0x00,
  311. 0x8b,0x30,0x00,0x00,0x7b,0x30,0x00,0x00,0x46,0x30,0x00,0x00,0x4c,0x30,0x00,0x00,
  312. 0x44,0x30,0x00,0x00,0x44,0x30,0x00,0x00,0x67,0x30,0x00,0x00,0x59,0x30,0x00,0x00,
  313. 0x02,0x30,0x00,0x00
  314. };
  315. // encoded by iconv
  316. static const unsigned char welcome_utf32be[212] =
  317. {
  318. 0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x63,
  319. 0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,
  320. 0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x6f,
  321. 0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x63,
  322. 0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x72,
  323. 0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x61,
  324. 0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x66,
  325. 0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x65,
  326. 0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x30,0x59,
  327. 0x00,0x00,0x30,0x50,0x00,0x00,0x8b,0x66,0x00,0x00,0x5b,0xdf,0x00,0x00,0x30,0x6b,
  328. 0x00,0x00,0x96,0xfb,0x00,0x00,0x8a,0x71,0x00,0x00,0x30,0x92,0x00,0x00,0x30,0x59,
  329. 0x00,0x00,0x30,0x8b,0x00,0x00,0x30,0x7b,0x00,0x00,0x30,0x46,0x00,0x00,0x30,0x4c,
  330. 0x00,0x00,0x30,0x44,0x00,0x00,0x30,0x44,0x00,0x00,0x30,0x67,0x00,0x00,0x30,0x59,
  331. 0x00,0x00,0x30,0x02
  332. };
  333. // encoded by iconv
  334. static const unsigned char welcome_cp932[71] =
  335. {
  336. 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63,
  337. 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65,
  338. 0x2e,0x20,0x20,0x82,0xb7,0x82,0xae,0x8c,0x78,0x8e,0x40,0x82,0xc9,0x93,0x64,0x98,
  339. 0x62,0x82,0xf0,0x82,0xb7,0x82,0xe9,0x82,0xd9,0x82,0xa4,0x82,0xaa,0x82,0xa2,0x82,
  340. 0xa2,0x82,0xc5,0x82,0xb7,0x81,0x42
  341. };
  342. #if wxBYTE_ORDER == wxBIG_ENDIAN
  343. #if SIZEOF_WCHAR_T == 2
  344. #define welcome_wchar_t welcome_utf16be
  345. #elif SIZEOF_WCHAR_T == 4
  346. #define welcome_wchar_t welcome_utf32be
  347. #endif
  348. #elif wxBYTE_ORDER == wxLITTLE_ENDIAN
  349. #if SIZEOF_WCHAR_T == 2
  350. #define welcome_wchar_t welcome_utf16le
  351. #elif SIZEOF_WCHAR_T == 4
  352. #define welcome_wchar_t welcome_utf32le
  353. #endif
  354. #endif
  355. void MBConvTestCase::UTF7Tests()
  356. {
  357. #if 0
  358. wxCSConv convUTF7(wxFONTENCODING_UTF7);
  359. #else
  360. wxMBConvUTF7 convUTF7;
  361. #endif
  362. TestDecoder
  363. (
  364. (const wchar_t*)welcome_wchar_t,
  365. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  366. (const char*)welcome_utf7_iconv,
  367. sizeof(welcome_utf7_iconv),
  368. convUTF7,
  369. 1
  370. );
  371. TestDecoder
  372. (
  373. (const wchar_t*)welcome_wchar_t,
  374. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  375. (const char*)welcome_utf7_wx,
  376. sizeof(welcome_utf7_wx),
  377. convUTF7,
  378. 1
  379. );
  380. #if 0
  381. // wxWidget's UTF-7 encoder generates different byte sequences than iconv's.
  382. // but both seem to be equally legal.
  383. // This test won't work and that's okay.
  384. TestEncoder
  385. (
  386. (const wchar_t*)welcome_wchar_t,
  387. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  388. (const char*)welcome_utf7_iconv,
  389. sizeof(welcome_utf7_iconv),
  390. convUTF7,
  391. 1
  392. );
  393. #endif
  394. TestEncoder
  395. (
  396. (const wchar_t*)welcome_wchar_t,
  397. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  398. (const char*)welcome_utf7_wx,
  399. sizeof(welcome_utf7_wx),
  400. convUTF7,
  401. 1
  402. );
  403. }
  404. void MBConvTestCase::UTF8Tests()
  405. {
  406. TestDecoder
  407. (
  408. (const wchar_t*)welcome_wchar_t,
  409. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  410. (const char*)welcome_utf8,
  411. sizeof(welcome_utf8),
  412. wxConvUTF8,
  413. 1
  414. );
  415. TestEncoder
  416. (
  417. (const wchar_t*)welcome_wchar_t,
  418. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  419. (const char*)welcome_utf8,
  420. sizeof(welcome_utf8),
  421. wxConvUTF8,
  422. 1
  423. );
  424. }
  425. void MBConvTestCase::UTF16LETests()
  426. {
  427. wxMBConvUTF16LE convUTF16LE;
  428. TestDecoder
  429. (
  430. (const wchar_t*)welcome_wchar_t,
  431. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  432. (const char*)welcome_utf16le,
  433. sizeof(welcome_utf16le),
  434. convUTF16LE,
  435. 2
  436. );
  437. TestEncoder
  438. (
  439. (const wchar_t*)welcome_wchar_t,
  440. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  441. (const char*)welcome_utf16le,
  442. sizeof(welcome_utf16le),
  443. convUTF16LE,
  444. 2
  445. );
  446. }
  447. void MBConvTestCase::UTF16BETests()
  448. {
  449. wxMBConvUTF16BE convUTF16BE;
  450. TestDecoder
  451. (
  452. (const wchar_t*)welcome_wchar_t,
  453. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  454. (const char*)welcome_utf16be,
  455. sizeof(welcome_utf16be),
  456. convUTF16BE,
  457. 2
  458. );
  459. TestEncoder
  460. (
  461. (const wchar_t*)welcome_wchar_t,
  462. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  463. (const char*)welcome_utf16be,
  464. sizeof(welcome_utf16be),
  465. convUTF16BE,
  466. 2
  467. );
  468. }
  469. void MBConvTestCase::UTF32LETests()
  470. {
  471. wxMBConvUTF32LE convUTF32LE;
  472. TestDecoder
  473. (
  474. (const wchar_t*)welcome_wchar_t,
  475. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  476. (const char*)welcome_utf32le,
  477. sizeof(welcome_utf32le),
  478. convUTF32LE,
  479. 4
  480. );
  481. TestEncoder
  482. (
  483. (const wchar_t*)welcome_wchar_t,
  484. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  485. (const char*)welcome_utf32le,
  486. sizeof(welcome_utf32le),
  487. convUTF32LE,
  488. 4
  489. );
  490. }
  491. void MBConvTestCase::UTF32BETests()
  492. {
  493. wxMBConvUTF32BE convUTF32BE;
  494. TestDecoder
  495. (
  496. (const wchar_t*)welcome_wchar_t,
  497. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  498. (const char*)welcome_utf32be,
  499. sizeof(welcome_utf32be),
  500. convUTF32BE,
  501. 4
  502. );
  503. TestEncoder
  504. (
  505. (const wchar_t*)welcome_wchar_t,
  506. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  507. (const char*)welcome_utf32be,
  508. sizeof(welcome_utf32be),
  509. convUTF32BE,
  510. 4
  511. );
  512. }
  513. void MBConvTestCase::CP932Tests()
  514. {
  515. wxCSConv convCP932( wxFONTENCODING_CP932 );
  516. TestDecoder
  517. (
  518. (const wchar_t*)welcome_wchar_t,
  519. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  520. (const char*)welcome_cp932,
  521. sizeof(welcome_cp932),
  522. convCP932,
  523. 1
  524. );
  525. TestEncoder
  526. (
  527. (const wchar_t*)welcome_wchar_t,
  528. sizeof(welcome_wchar_t)/sizeof(wchar_t),
  529. (const char*)welcome_cp932,
  530. sizeof(welcome_cp932),
  531. convCP932,
  532. 1
  533. );
  534. }
  535. // a character sequence encoded as iso8859-1 (iconv)
  536. static const unsigned char iso8859_1[251] =
  537. {
  538. 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,
  539. 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,
  540. 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,
  541. 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,
  542. 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,
  543. 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,
  544. 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,
  545. 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84,
  546. 0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,
  547. 0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xa1,0xa2,0xa3,0xa4,
  548. 0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,
  549. 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4,
  550. 0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,
  551. 0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4,
  552. 0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,
  553. 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff
  554. };
  555. // the above character sequence encoded as UTF-8 (iconv)
  556. static const unsigned char iso8859_1_utf8[379] =
  557. {
  558. 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,
  559. 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,
  560. 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,
  561. 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,
  562. 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,
  563. 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,
  564. 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,
  565. 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xc2,0x80,0xc2,0x81,0xc2,
  566. 0x82,0xc2,0x83,0xc2,0x84,0xc2,0x85,0xc2,0x86,0xc2,0x87,0xc2,0x88,0xc2,0x89,0xc2,
  567. 0x8a,0xc2,0x8b,0xc2,0x8c,0xc2,0x8d,0xc2,0x8e,0xc2,0x8f,0xc2,0x90,0xc2,0x91,0xc2,
  568. 0x92,0xc2,0x93,0xc2,0x94,0xc2,0x95,0xc2,0x96,0xc2,0x97,0xc2,0x98,0xc2,0x99,0xc2,
  569. 0x9a,0xc2,0x9b,0xc2,0x9c,0xc2,0x9d,0xc2,0x9e,0xc2,0x9f,0xc2,0xa0,0xc2,0xa1,0xc2,
  570. 0xa2,0xc2,0xa3,0xc2,0xa4,0xc2,0xa5,0xc2,0xa6,0xc2,0xa7,0xc2,0xa8,0xc2,0xa9,0xc2,
  571. 0xaa,0xc2,0xab,0xc2,0xac,0xc2,0xad,0xc2,0xae,0xc2,0xaf,0xc2,0xb0,0xc2,0xb1,0xc2,
  572. 0xb2,0xc2,0xb3,0xc2,0xb4,0xc2,0xb5,0xc2,0xb6,0xc2,0xb7,0xc2,0xb8,0xc2,0xb9,0xc2,
  573. 0xba,0xc2,0xbb,0xc2,0xbc,0xc2,0xbd,0xc2,0xbe,0xc2,0xbf,0xc3,0x80,0xc3,0x81,0xc3,
  574. 0x82,0xc3,0x83,0xc3,0x84,0xc3,0x85,0xc3,0x86,0xc3,0x87,0xc3,0x88,0xc3,0x89,0xc3,
  575. 0x8a,0xc3,0x8b,0xc3,0x8c,0xc3,0x8d,0xc3,0x8e,0xc3,0x8f,0xc3,0x90,0xc3,0x91,0xc3,
  576. 0x92,0xc3,0x93,0xc3,0x94,0xc3,0x95,0xc3,0x96,0xc3,0x97,0xc3,0x98,0xc3,0x99,0xc3,
  577. 0x9a,0xc3,0x9b,0xc3,0x9c,0xc3,0x9d,0xc3,0x9e,0xc3,0x9f,0xc3,0xa0,0xc3,0xa1,0xc3,
  578. 0xa2,0xc3,0xa3,0xc3,0xa4,0xc3,0xa5,0xc3,0xa6,0xc3,0xa7,0xc3,0xa8,0xc3,0xa9,0xc3,
  579. 0xaa,0xc3,0xab,0xc3,0xac,0xc3,0xad,0xc3,0xae,0xc3,0xaf,0xc3,0xb0,0xc3,0xb1,0xc3,
  580. 0xb2,0xc3,0xb3,0xc3,0xb4,0xc3,0xb5,0xc3,0xb6,0xc3,0xb7,0xc3,0xb8,0xc3,0xb9,0xc3,
  581. 0xba,0xc3,0xbb,0xc3,0xbc,0xc3,0xbd,0xc3,0xbe,0xc3,0xbf
  582. };
  583. // a character sequence encoded as CP1252 (iconv)
  584. static const unsigned char CP1252[246] =
  585. {
  586. 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,
  587. 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,
  588. 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,
  589. 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,
  590. 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,
  591. 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,
  592. 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,
  593. 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xa0,0xa1,0xa2,0xa3,0xa4,
  594. 0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,
  595. 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4,
  596. 0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,
  597. 0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4,
  598. 0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,
  599. 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff,0x8c,0x9c,0x8a,0x9a,0x9f,
  600. 0x8e,0x9e,0x83,0x88,0x98,0x96,0x97,0x91,0x92,0x82,0x93,0x94,0x84,0x86,0x87,0x95,
  601. 0x85,0x89,0x8b,0x9b,0x80,0x99
  602. };
  603. // the above character sequence encoded as UTF-8 (iconv)
  604. static const unsigned char CP1252_utf8[386] =
  605. {
  606. 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,
  607. 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,
  608. 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,
  609. 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,
  610. 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,
  611. 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,
  612. 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,
  613. 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xc2,0xa0,0xc2,0xa1,0xc2,
  614. 0xa2,0xc2,0xa3,0xc2,0xa4,0xc2,0xa5,0xc2,0xa6,0xc2,0xa7,0xc2,0xa8,0xc2,0xa9,0xc2,
  615. 0xaa,0xc2,0xab,0xc2,0xac,0xc2,0xad,0xc2,0xae,0xc2,0xaf,0xc2,0xb0,0xc2,0xb1,0xc2,
  616. 0xb2,0xc2,0xb3,0xc2,0xb4,0xc2,0xb5,0xc2,0xb6,0xc2,0xb7,0xc2,0xb8,0xc2,0xb9,0xc2,
  617. 0xba,0xc2,0xbb,0xc2,0xbc,0xc2,0xbd,0xc2,0xbe,0xc2,0xbf,0xc3,0x80,0xc3,0x81,0xc3,
  618. 0x82,0xc3,0x83,0xc3,0x84,0xc3,0x85,0xc3,0x86,0xc3,0x87,0xc3,0x88,0xc3,0x89,0xc3,
  619. 0x8a,0xc3,0x8b,0xc3,0x8c,0xc3,0x8d,0xc3,0x8e,0xc3,0x8f,0xc3,0x90,0xc3,0x91,0xc3,
  620. 0x92,0xc3,0x93,0xc3,0x94,0xc3,0x95,0xc3,0x96,0xc3,0x97,0xc3,0x98,0xc3,0x99,0xc3,
  621. 0x9a,0xc3,0x9b,0xc3,0x9c,0xc3,0x9d,0xc3,0x9e,0xc3,0x9f,0xc3,0xa0,0xc3,0xa1,0xc3,
  622. 0xa2,0xc3,0xa3,0xc3,0xa4,0xc3,0xa5,0xc3,0xa6,0xc3,0xa7,0xc3,0xa8,0xc3,0xa9,0xc3,
  623. 0xaa,0xc3,0xab,0xc3,0xac,0xc3,0xad,0xc3,0xae,0xc3,0xaf,0xc3,0xb0,0xc3,0xb1,0xc3,
  624. 0xb2,0xc3,0xb3,0xc3,0xb4,0xc3,0xb5,0xc3,0xb6,0xc3,0xb7,0xc3,0xb8,0xc3,0xb9,0xc3,
  625. 0xba,0xc3,0xbb,0xc3,0xbc,0xc3,0xbd,0xc3,0xbe,0xc3,0xbf,0xc5,0x92,0xc5,0x93,0xc5,
  626. 0xa0,0xc5,0xa1,0xc5,0xb8,0xc5,0xbd,0xc5,0xbe,0xc6,0x92,0xcb,0x86,0xcb,0x9c,0xe2,
  627. 0x80,0x93,0xe2,0x80,0x94,0xe2,0x80,0x98,0xe2,0x80,0x99,0xe2,0x80,0x9a,0xe2,0x80,
  628. 0x9c,0xe2,0x80,0x9d,0xe2,0x80,0x9e,0xe2,0x80,0xa0,0xe2,0x80,0xa1,0xe2,0x80,0xa2,
  629. 0xe2,0x80,0xa6,0xe2,0x80,0xb0,0xe2,0x80,0xb9,0xe2,0x80,0xba,0xe2,0x82,0xac,0xe2,
  630. 0x84,0xa2
  631. };
  632. // this is unused currently so avoid warnings about this
  633. #if 0
  634. // a character sequence encoded as iso8859-5 (iconv)
  635. static const unsigned char iso8859_5[251] =
  636. {
  637. 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,
  638. 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,
  639. 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,
  640. 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,
  641. 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,
  642. 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,
  643. 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,
  644. 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84,
  645. 0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,
  646. 0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xfd,0xad,0xa1,0xa2,
  647. 0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,
  648. 0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,
  649. 0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,
  650. 0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,
  651. 0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf1,0xf2,0xf3,0xf4,
  652. 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfe,0xff,0xf0
  653. };
  654. // the above character sequence encoded as UTF-8 (iconv)
  655. static const unsigned char iso8859_5_utf8[380] =
  656. {
  657. 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,
  658. 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24,
  659. 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,
  660. 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,
  661. 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,
  662. 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,
  663. 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,
  664. 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xc2,0x80,0xc2,0x81,0xc2,
  665. 0x82,0xc2,0x83,0xc2,0x84,0xc2,0x85,0xc2,0x86,0xc2,0x87,0xc2,0x88,0xc2,0x89,0xc2,
  666. 0x8a,0xc2,0x8b,0xc2,0x8c,0xc2,0x8d,0xc2,0x8e,0xc2,0x8f,0xc2,0x90,0xc2,0x91,0xc2,
  667. 0x92,0xc2,0x93,0xc2,0x94,0xc2,0x95,0xc2,0x96,0xc2,0x97,0xc2,0x98,0xc2,0x99,0xc2,
  668. 0x9a,0xc2,0x9b,0xc2,0x9c,0xc2,0x9d,0xc2,0x9e,0xc2,0x9f,0xc2,0xa0,0xc2,0xa7,0xc2,
  669. 0xad,0xd0,0x81,0xd0,0x82,0xd0,0x83,0xd0,0x84,0xd0,0x85,0xd0,0x86,0xd0,0x87,0xd0,
  670. 0x88,0xd0,0x89,0xd0,0x8a,0xd0,0x8b,0xd0,0x8c,0xd0,0x8e,0xd0,0x8f,0xd0,0x90,0xd0,
  671. 0x91,0xd0,0x92,0xd0,0x93,0xd0,0x94,0xd0,0x95,0xd0,0x96,0xd0,0x97,0xd0,0x98,0xd0,
  672. 0x99,0xd0,0x9a,0xd0,0x9b,0xd0,0x9c,0xd0,0x9d,0xd0,0x9e,0xd0,0x9f,0xd0,0xa0,0xd0,
  673. 0xa1,0xd0,0xa2,0xd0,0xa3,0xd0,0xa4,0xd0,0xa5,0xd0,0xa6,0xd0,0xa7,0xd0,0xa8,0xd0,
  674. 0xa9,0xd0,0xaa,0xd0,0xab,0xd0,0xac,0xd0,0xad,0xd0,0xae,0xd0,0xaf,0xd0,0xb0,0xd0,
  675. 0xb1,0xd0,0xb2,0xd0,0xb3,0xd0,0xb4,0xd0,0xb5,0xd0,0xb6,0xd0,0xb7,0xd0,0xb8,0xd0,
  676. 0xb9,0xd0,0xba,0xd0,0xbb,0xd0,0xbc,0xd0,0xbd,0xd0,0xbe,0xd0,0xbf,0xd1,0x80,0xd1,
  677. 0x81,0xd1,0x82,0xd1,0x83,0xd1,0x84,0xd1,0x85,0xd1,0x86,0xd1,0x87,0xd1,0x88,0xd1,
  678. 0x89,0xd1,0x8a,0xd1,0x8b,0xd1,0x8c,0xd1,0x8d,0xd1,0x8e,0xd1,0x8f,0xd1,0x91,0xd1,
  679. 0x92,0xd1,0x93,0xd1,0x94,0xd1,0x95,0xd1,0x96,0xd1,0x97,0xd1,0x98,0xd1,0x99,0xd1,
  680. 0x9a,0xd1,0x9b,0xd1,0x9c,0xd1,0x9e,0xd1,0x9f,0xe2,0x84,0x96
  681. };
  682. #endif // 0
  683. // DecodeUTF8
  684. // decodes the specified *unterminated* UTF-8 byte array
  685. wxWCharBuffer DecodeUTF8(
  686. const void* data, // an unterminated UTF-8 encoded byte array
  687. size_t size // the byte length of data
  688. )
  689. {
  690. // the decoder requires a null terminated buffer.
  691. // the input data is not null terminated.
  692. // copy to null terminated buffer
  693. wxCharBuffer nullTerminated( size+1 );
  694. memcpy( nullTerminated.data(), data, size );
  695. nullTerminated.data()[size] = 0;
  696. return wxConvUTF8.cMB2WC(nullTerminated.data());
  697. }
  698. // tests the encoding and decoding capability of an wxMBConv object
  699. //
  700. // decodes the utf-8 bytes into wide characters
  701. // encodes the wide characters to compare against input multiBuffer
  702. // decodes the multiBuffer to compare against wide characters
  703. // decodes the multiBuffer into wide characters
  704. void MBConvTestCase::TestCoder(
  705. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  706. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  707. const char* utf8Buffer, // the same character sequence as multiBuffer, encoded as UTF-8
  708. size_t utf8Bytes, // the byte length of the UTF-8 encoded character sequence
  709. wxMBConv& converter, // the wxMBConv object thta can decode multiBuffer into a wide character sequence
  710. int sizeofNull // the number of bytes occupied by a terminating null in the converter's encoding
  711. )
  712. {
  713. // wide character size and endian-ess varies from platform to platform
  714. // compiler support for wide character literals varies from compiler to compiler
  715. // so we should store the wide character version as UTF-8 and depend on
  716. // the UTF-8 converter's ability to decode it to platform specific wide characters
  717. // this test is invalid if the UTF-8 converter can't decode
  718. const wxWCharBuffer wideBuffer(DecodeUTF8(utf8Buffer, utf8Bytes));
  719. const size_t wideChars = wxWcslen(wideBuffer);
  720. TestDecoder
  721. (
  722. wideBuffer.data(),
  723. wideChars,
  724. multiBuffer,
  725. multiBytes,
  726. converter,
  727. sizeofNull
  728. );
  729. TestEncoder
  730. (
  731. wideBuffer.data(),
  732. wideChars,
  733. multiBuffer,
  734. multiBytes,
  735. converter,
  736. sizeofNull
  737. );
  738. }
  739. WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const char* name );
  740. void MBConvTestCase::FontmapTests()
  741. {
  742. #ifdef wxUSE_FONTMAP
  743. wxMBConv* converter = new_wxMBConv_wxwin("CP1252");
  744. if ( !converter )
  745. {
  746. return;
  747. }
  748. TestCoder(
  749. (const char*)CP1252,
  750. sizeof(CP1252),
  751. (const char*)CP1252_utf8,
  752. sizeof(CP1252_utf8),
  753. *converter,
  754. 1
  755. );
  756. delete converter;
  757. #endif
  758. }
  759. void MBConvTestCase::BufSize()
  760. {
  761. wxCSConv conv1251(wxT("CP1251"));
  762. CPPUNIT_ASSERT( conv1251.IsOk() );
  763. const char *cp1251text =
  764. "\313\301\326\305\324\323\321 \325\304\301\336\316\331\315";
  765. const size_t lenW = conv1251.MB2WC(NULL, cp1251text, 0);
  766. CPPUNIT_ASSERT_EQUAL( strlen(cp1251text), lenW );
  767. wxWCharBuffer wbuf(lenW + 1); // allocates lenW + 2 characters
  768. wbuf.data()[lenW + 1] = L'!';
  769. // lenW is not enough because it's the length and we need the size
  770. CPPUNIT_ASSERT_EQUAL(
  771. wxCONV_FAILED, conv1251.MB2WC(wbuf.data(), cp1251text, lenW) );
  772. // lenW+1 is just fine
  773. CPPUNIT_ASSERT(
  774. conv1251.MB2WC(wbuf.data(), cp1251text, lenW + 1) != wxCONV_FAILED );
  775. // of course, greater values work too
  776. CPPUNIT_ASSERT(
  777. conv1251.MB2WC(wbuf.data(), cp1251text, lenW + 2) != wxCONV_FAILED );
  778. // but they shouldn't write more stuff to the buffer
  779. CPPUNIT_ASSERT_EQUAL( L'!', wbuf[lenW + 1] );
  780. // test in the other direction too, using an encoding with multibyte NUL
  781. wxCSConv convUTF16(wxT("UTF-16LE"));
  782. CPPUNIT_ASSERT( convUTF16.IsOk() );
  783. const wchar_t *utf16text = L"Hello";
  784. const size_t lenMB = convUTF16.WC2MB(NULL, utf16text, 0);
  785. CPPUNIT_ASSERT_EQUAL( wcslen(utf16text)*2, lenMB );
  786. wxCharBuffer buf(lenMB + 2); // it only adds 1 for NUL on its own, we need 2
  787. // for NUL and an extra one for the guard byte
  788. buf.data()[lenMB + 2] = '?';
  789. CPPUNIT_ASSERT_EQUAL(
  790. wxCONV_FAILED, convUTF16.WC2MB(buf.data(), utf16text, lenMB) );
  791. CPPUNIT_ASSERT_EQUAL(
  792. wxCONV_FAILED, convUTF16.WC2MB(buf.data(), utf16text, lenMB + 1) );
  793. CPPUNIT_ASSERT(
  794. convUTF16.WC2MB(buf.data(), utf16text, lenMB + 2) != wxCONV_FAILED );
  795. CPPUNIT_ASSERT(
  796. convUTF16.WC2MB(buf.data(), utf16text, lenMB + 3) != wxCONV_FAILED );
  797. CPPUNIT_ASSERT_EQUAL( '?', buf[lenMB + 2] );
  798. }
  799. void MBConvTestCase::FromWCharTests()
  800. {
  801. wxCSConv conv950("CP950");
  802. char mbuf[10];
  803. // U+4e00 is 2 bytes (0xa4 0x40) in cp950
  804. wchar_t wbuf[] = { 0x4e00, 0, 0x4e00, 0 };
  805. // test simple ASCII text
  806. memset(mbuf, '!', sizeof(mbuf));
  807. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 0, L"a", 1));
  808. CPPUNIT_ASSERT_EQUAL( '!', mbuf[0]);
  809. memset(mbuf, '!', sizeof(mbuf));
  810. CPPUNIT_ASSERT_EQUAL( 1, conv950.FromWChar(mbuf, 1, L"a", 1));
  811. CPPUNIT_ASSERT_EQUAL( 'a', mbuf[0]);
  812. CPPUNIT_ASSERT_EQUAL( '!', mbuf[1]);
  813. memset(mbuf, '!', sizeof(mbuf));
  814. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 1, L"a", 2));
  815. memset(mbuf, '!', sizeof(mbuf));
  816. CPPUNIT_ASSERT_EQUAL( 2, conv950.FromWChar(mbuf, 2, L"a", 2));
  817. CPPUNIT_ASSERT_EQUAL( 'a', mbuf[0]);
  818. CPPUNIT_ASSERT_EQUAL( '\0', mbuf[1]);
  819. CPPUNIT_ASSERT_EQUAL( '!', mbuf[2]);
  820. // test non-ASCII text, 1 wchar -> 2 char
  821. memset(mbuf, '!', sizeof(mbuf));
  822. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 0, wbuf, 1));
  823. memset(mbuf, '!', sizeof(mbuf));
  824. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 1, wbuf, 1));
  825. memset(mbuf, '!', sizeof(mbuf));
  826. CPPUNIT_ASSERT_EQUAL( 2, conv950.FromWChar(mbuf, 2, wbuf, 1));
  827. CPPUNIT_ASSERT_EQUAL( '!', mbuf[2]);
  828. memset(mbuf, '!', sizeof(mbuf));
  829. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 2, wbuf, 2));
  830. memset(mbuf, '!', sizeof(mbuf));
  831. CPPUNIT_ASSERT_EQUAL( 3, conv950.FromWChar(mbuf, 3, wbuf, 2));
  832. CPPUNIT_ASSERT_EQUAL( '\0', mbuf[2]);
  833. CPPUNIT_ASSERT_EQUAL( '!', mbuf[3]);
  834. // test text with embedded NUL-character and srcLen specified
  835. memset(mbuf, '!', sizeof(mbuf));
  836. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 3, wbuf, 3));
  837. memset(mbuf, '!', sizeof(mbuf));
  838. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 4, wbuf, 3));
  839. CPPUNIT_ASSERT_EQUAL( 5, conv950.FromWChar(mbuf, 5, wbuf, 3));
  840. CPPUNIT_ASSERT_EQUAL( '\0', mbuf[2]);
  841. CPPUNIT_ASSERT_EQUAL( '!', mbuf[5]);
  842. memset(mbuf, '!', sizeof(mbuf));
  843. CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 5, wbuf, 4));
  844. memset(mbuf, '!', sizeof(mbuf));
  845. CPPUNIT_ASSERT_EQUAL( 6, conv950.FromWChar(mbuf, 6, wbuf, 4));
  846. CPPUNIT_ASSERT_EQUAL( '\0', mbuf[2]);
  847. CPPUNIT_ASSERT_EQUAL( '\0', mbuf[5]);
  848. CPPUNIT_ASSERT_EQUAL( '!', mbuf[6]);
  849. }
  850. WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_iconv( const char* name );
  851. void MBConvTestCase::IconvTests()
  852. {
  853. #ifdef HAVE_ICONV
  854. wxMBConv* converter = new_wxMBConv_iconv("CP932");
  855. if ( !converter )
  856. {
  857. return;
  858. }
  859. TestCoder(
  860. (const char*)welcome_cp932,
  861. sizeof(welcome_cp932),
  862. (const char*)welcome_utf8,
  863. sizeof(welcome_utf8),
  864. *converter,
  865. 1
  866. );
  867. delete converter;
  868. #endif
  869. }
  870. void MBConvTestCase::Latin1Tests()
  871. {
  872. TestCoder(
  873. (const char*)iso8859_1,
  874. sizeof(iso8859_1),
  875. (const char*)iso8859_1_utf8,
  876. sizeof(iso8859_1_utf8),
  877. wxConvISO8859_1,
  878. 1
  879. );
  880. static const char nulstr[] = "foo\0bar\0";
  881. static const size_t mbLen = WXSIZEOF(nulstr) - 1;
  882. size_t wcLen;
  883. wxConvISO8859_1.cMB2WC(nulstr, mbLen, &wcLen);
  884. CPPUNIT_ASSERT_EQUAL( mbLen, wcLen );
  885. }
  886. void MBConvTestCase::CP1252Tests()
  887. {
  888. wxCSConv convCP1252( wxFONTENCODING_CP1252 );
  889. TestCoder(
  890. (const char*)CP1252,
  891. sizeof(CP1252),
  892. (const char*)CP1252_utf8,
  893. sizeof(CP1252_utf8),
  894. convCP1252,
  895. 1
  896. );
  897. }
  898. void MBConvTestCase::LibcTests()
  899. {
  900. // The locale name are OS-dependent so this test is done only under Windows
  901. // when using MSVC (surprisingly it fails with MinGW, even though it's
  902. // supposed to use the same CRT -- no idea why and unfortunately gdb is too
  903. // flaky to debug it)
  904. #ifdef __VISUALC__
  905. LocaleSetter loc("English_United States.1252");
  906. wxMBConvLibc convLibc;
  907. TestCoder(
  908. (const char*)CP1252,
  909. sizeof(CP1252),
  910. (const char*)CP1252_utf8,
  911. sizeof(CP1252_utf8),
  912. convLibc,
  913. 1
  914. );
  915. #endif // __VISUALC__
  916. }
  917. // verifies that the specified mb sequences decode to the specified wc sequence
  918. void MBConvTestCase::TestDecoder(
  919. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  920. size_t wideChars, // the number of wide characters at wideBuffer
  921. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  922. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  923. wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence
  924. int sizeofNull // number of bytes occupied by terminating null in this encoding
  925. )
  926. {
  927. const unsigned UNINITIALIZED = 0xcd;
  928. // copy the input bytes into a null terminated buffer
  929. wxCharBuffer inputCopy( multiBytes+sizeofNull );
  930. memcpy( inputCopy.data(), multiBuffer, multiBytes );
  931. memset( &inputCopy.data()[multiBytes], 0, sizeofNull );
  932. // calculate the output size
  933. size_t outputWritten = converter.MB2WC
  934. (
  935. 0,
  936. (const char*)inputCopy.data(),
  937. 0
  938. );
  939. // make sure the correct output length was calculated
  940. WX_ASSERT_EQUAL_MESSAGE
  941. (
  942. ("while converting \"%s\"", multiBuffer),
  943. wideChars,
  944. outputWritten
  945. );
  946. // convert the string
  947. size_t guardChars = 8; // to make sure we're not overrunning the output buffer
  948. size_t nullCharacters = 1;
  949. size_t outputBufferChars = outputWritten + nullCharacters + guardChars;
  950. wxWCharBuffer outputBuffer(outputBufferChars);
  951. memset( outputBuffer.data(), UNINITIALIZED, outputBufferChars*sizeof(wchar_t) );
  952. outputWritten = converter.MB2WC
  953. (
  954. outputBuffer.data(),
  955. (const char*)inputCopy.data(),
  956. outputBufferChars
  957. );
  958. // make sure the correct number of characters were outputs
  959. CPPUNIT_ASSERT_EQUAL( wideChars, outputWritten );
  960. // make sure the characters generated are correct
  961. CPPUNIT_ASSERT( 0 == memcmp( outputBuffer, wideBuffer, wideChars*sizeof(wchar_t) ) );
  962. // the output buffer should be null terminated
  963. CPPUNIT_ASSERT( outputBuffer[outputWritten] == 0 );
  964. // make sure the rest of the output buffer is untouched
  965. for ( size_t i = (wideChars+1)*sizeof(wchar_t); i < (outputBufferChars*sizeof(wchar_t)); i++ )
  966. {
  967. CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == UNINITIALIZED );
  968. }
  969. #if wxUSE_UNICODE && wxUSE_STREAMS
  970. TestStreamDecoder( wideBuffer, wideChars, multiBuffer, multiBytes, converter );
  971. #endif
  972. }
  973. // verifies that the specified wc sequences encodes to the specified mb sequence
  974. void MBConvTestCase::TestEncoder(
  975. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  976. size_t wideChars, // the number of wide characters at wideBuffer
  977. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  978. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  979. wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence
  980. int sizeofNull // number of bytes occupied by terminating null in this encoding
  981. )
  982. {
  983. const unsigned UNINITIALIZED = 0xcd;
  984. // copy the input bytes into a null terminated buffer
  985. wxWCharBuffer inputCopy( wideChars + 1 );
  986. memcpy( inputCopy.data(), wideBuffer, (wideChars*sizeof(wchar_t)) );
  987. inputCopy.data()[wideChars] = 0;
  988. // calculate the output size
  989. size_t outputWritten = converter.WC2MB
  990. (
  991. 0,
  992. (const wchar_t*)inputCopy.data(),
  993. 0
  994. );
  995. // make sure the correct output length was calculated
  996. CPPUNIT_ASSERT_EQUAL( multiBytes, outputWritten );
  997. // convert the string
  998. size_t guardBytes = 8; // to make sure we're not overrunning the output buffer
  999. size_t outputBufferSize = outputWritten + sizeofNull + guardBytes;
  1000. wxCharBuffer outputBuffer(outputBufferSize);
  1001. memset( outputBuffer.data(), UNINITIALIZED, outputBufferSize );
  1002. outputWritten = converter.WC2MB
  1003. (
  1004. outputBuffer.data(),
  1005. (const wchar_t*)inputCopy.data(),
  1006. outputBufferSize
  1007. );
  1008. // make sure the correct number of characters were output
  1009. CPPUNIT_ASSERT_EQUAL( multiBytes, outputWritten );
  1010. // make sure the characters generated are correct
  1011. CPPUNIT_ASSERT( 0 == memcmp( outputBuffer, multiBuffer, multiBytes ) );
  1012. size_t i;
  1013. // the output buffer should be null terminated
  1014. for ( i = multiBytes; i < multiBytes + sizeofNull; i++ )
  1015. {
  1016. CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == 0 );
  1017. }
  1018. // make sure the rest of the output buffer is untouched
  1019. for ( i = multiBytes + sizeofNull; i < outputBufferSize; i++ )
  1020. {
  1021. CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == UNINITIALIZED );
  1022. }
  1023. #if wxUSE_UNICODE && wxUSE_STREAMS
  1024. TestStreamEncoder( wideBuffer, wideChars, multiBuffer, multiBytes, converter );
  1025. #endif
  1026. }
  1027. #if wxUSE_UNICODE && wxUSE_STREAMS
  1028. // use wxTextInputStream to exercise wxMBConv interface
  1029. // (this reveals some bugs in certain wxMBConv subclasses)
  1030. void MBConvTestCase::TestStreamDecoder(
  1031. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  1032. size_t wideChars, // the number of wide characters at wideBuffer
  1033. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  1034. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  1035. wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence
  1036. )
  1037. {
  1038. // this isn't meant to test wxMemoryInputStream or wxTextInputStream
  1039. // it's meant to test the way wxTextInputStream uses wxMBConv
  1040. // (which has exposed some problems with wxMBConv)
  1041. wxMemoryInputStream memoryInputStream( multiBuffer, multiBytes );
  1042. wxTextInputStream textInputStream( memoryInputStream, wxT(""), converter );
  1043. for ( size_t i = 0; i < wideChars; i++ )
  1044. {
  1045. wxChar wc = textInputStream.GetChar();
  1046. CPPUNIT_ASSERT_EQUAL_MESSAGE(
  1047. std::string(wxString::Format("At position %lu", (unsigned long)i)),
  1048. wideBuffer[i],
  1049. wc
  1050. );
  1051. }
  1052. CPPUNIT_ASSERT( 0 == textInputStream.GetChar() );
  1053. CPPUNIT_ASSERT( memoryInputStream.Eof() );
  1054. }
  1055. #endif
  1056. #if wxUSE_UNICODE && wxUSE_STREAMS
  1057. // use wxTextInputStream to exercise wxMBConv interface
  1058. // (this reveals some bugs in certain wxMBConv subclasses)
  1059. void MBConvTestCase::TestStreamEncoder(
  1060. const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t
  1061. size_t wideChars, // the number of wide characters at wideBuffer
  1062. const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter"
  1063. size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter"
  1064. wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence
  1065. )
  1066. {
  1067. // this isn't meant to test wxMemoryOutputStream or wxTextOutputStream
  1068. // it's meant to test the way wxTextOutputStream uses wxMBConv
  1069. // (which has exposed some problems with wxMBConv)
  1070. wxMemoryOutputStream memoryOutputStream;
  1071. // wxEOL_UNIX will pass \n \r unchanged
  1072. wxTextOutputStream textOutputStream( memoryOutputStream, wxEOL_UNIX, converter );
  1073. for ( size_t i = 0; i < wideChars; i++ )
  1074. {
  1075. textOutputStream.PutChar( wideBuffer[i] );
  1076. }
  1077. textOutputStream.Flush();
  1078. CPPUNIT_ASSERT_EQUAL( multiBytes, size_t(memoryOutputStream.TellO()) );
  1079. wxCharBuffer copy( memoryOutputStream.TellO() );
  1080. memoryOutputStream.CopyTo( copy.data(), memoryOutputStream.TellO());
  1081. CPPUNIT_ASSERT_EQUAL( 0, memcmp( copy.data(), multiBuffer, multiBytes ) );
  1082. }
  1083. #endif
  1084. // ----------------------------------------------------------------------------
  1085. // UTF-8 tests
  1086. // ----------------------------------------------------------------------------
  1087. #ifdef HAVE_WCHAR_H
  1088. // Check that 'charSequence' translates to 'wideSequence' and back.
  1089. // Invalid sequences can be tested by giving NULL for 'wideSequence'. Even
  1090. // invalid sequences should roundtrip when an option is given and this is
  1091. // checked.
  1092. //
  1093. void MBConvTestCase::UTF8(const char *charSequence,
  1094. const wchar_t *wideSequence)
  1095. {
  1096. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_NOT);
  1097. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA);
  1098. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL);
  1099. }
  1100. // Use this alternative when 'charSequence' contains a PUA character. Such
  1101. // sequences should still roundtrip ok, and this is checked.
  1102. //
  1103. void MBConvTestCase::UTF8PUA(const char *charSequence,
  1104. const wchar_t *wideSequence)
  1105. {
  1106. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_NOT);
  1107. UTF8(charSequence, NULL, wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA);
  1108. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL);
  1109. }
  1110. // Use this alternative when 'charSequence' contains an octal escape sequence.
  1111. // Such sequences should still roundtrip ok, and this is checked.
  1112. //
  1113. void MBConvTestCase::UTF8Octal(const char *charSequence,
  1114. const wchar_t *wideSequence)
  1115. {
  1116. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_NOT);
  1117. UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA);
  1118. UTF8(charSequence, NULL, wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL);
  1119. }
  1120. // in case wcscpy is missing
  1121. //
  1122. static wchar_t *wx_wcscpy(wchar_t *dest, const wchar_t *src)
  1123. {
  1124. wchar_t *d = dest;
  1125. while ((*d++ = *src++) != 0)
  1126. ;
  1127. return dest;
  1128. }
  1129. // in case wcscat is missing
  1130. //
  1131. static wchar_t *wx_wcscat(wchar_t *dest, const wchar_t *src)
  1132. {
  1133. wchar_t *d = dest;
  1134. while (*d)
  1135. d++;
  1136. while ((*d++ = *src++) != 0)
  1137. ;
  1138. return dest;
  1139. }
  1140. // in case wcscmp is missing
  1141. //
  1142. static int wx_wcscmp(const wchar_t *s1, const wchar_t *s2)
  1143. {
  1144. while (*s1 == *s2 && *s1 != 0)
  1145. {
  1146. s1++;
  1147. s2++;
  1148. }
  1149. return *s1 - *s2;
  1150. }
  1151. // in case wcslen is missing
  1152. //
  1153. static size_t wx_wcslen(const wchar_t *s)
  1154. {
  1155. const wchar_t *t = s;
  1156. while (*t != 0)
  1157. t++;
  1158. return t - s;
  1159. }
  1160. // include the option in the error messages so it's possible to see which
  1161. // test failed
  1162. #define UTF8ASSERT(expr) CPPUNIT_ASSERT_MESSAGE(#expr + errmsg, expr)
  1163. // The test implementation
  1164. //
  1165. void MBConvTestCase::UTF8(const char *charSequence,
  1166. const wchar_t *wideSequence,
  1167. int option)
  1168. {
  1169. const size_t BUFSIZE = 128;
  1170. wxASSERT(strlen(charSequence) * 3 + 10 < BUFSIZE);
  1171. char bytes[BUFSIZE];
  1172. // include the option in the error messages so it's possible to see
  1173. // which test failed
  1174. sprintf(bytes, " (with option == %d)", option);
  1175. std::string errmsg(bytes);
  1176. // put the charSequence at the start, middle and end of a string
  1177. strcpy(bytes, charSequence);
  1178. strcat(bytes, "ABC");
  1179. strcat(bytes, charSequence);
  1180. strcat(bytes, "XYZ");
  1181. strcat(bytes, charSequence);
  1182. // translate it into wide characters
  1183. wxMBConvUTF8 utf8(option);
  1184. wchar_t widechars[BUFSIZE];
  1185. size_t lenResult = utf8.MB2WC(NULL, bytes, 0);
  1186. size_t result = utf8.MB2WC(widechars, bytes, BUFSIZE);
  1187. UTF8ASSERT(result == lenResult);
  1188. // check we got the expected result
  1189. if (wideSequence) {
  1190. UTF8ASSERT(result != (size_t)-1);
  1191. wxASSERT(result < BUFSIZE);
  1192. wchar_t expected[BUFSIZE];
  1193. wx_wcscpy(expected, wideSequence);
  1194. wx_wcscat(expected, L"ABC");
  1195. wx_wcscat(expected, wideSequence);
  1196. wx_wcscat(expected, L"XYZ");
  1197. wx_wcscat(expected, wideSequence);
  1198. UTF8ASSERT(wx_wcscmp(widechars, expected) == 0);
  1199. UTF8ASSERT(wx_wcslen(widechars) == result);
  1200. }
  1201. else {
  1202. // If 'wideSequence' is NULL, then the result is expected to be
  1203. // invalid. Normally that is as far as we can go, but if there is an
  1204. // option then the conversion should succeed anyway, and it should be
  1205. // possible to translate back to the original
  1206. if (!option) {
  1207. UTF8ASSERT(result == (size_t)-1);
  1208. return;
  1209. }
  1210. else {
  1211. UTF8ASSERT(result != (size_t)-1);
  1212. }
  1213. }
  1214. // translate it back and check we get the original
  1215. char bytesAgain[BUFSIZE];
  1216. size_t lenResultAgain = utf8.WC2MB(NULL, widechars, 0);
  1217. size_t resultAgain = utf8.WC2MB(bytesAgain, widechars, BUFSIZE);
  1218. UTF8ASSERT(resultAgain == lenResultAgain);
  1219. UTF8ASSERT(resultAgain != (size_t)-1);
  1220. wxASSERT(resultAgain < BUFSIZE);
  1221. UTF8ASSERT(strcmp(bytes, bytesAgain) == 0);
  1222. UTF8ASSERT(strlen(bytesAgain) == resultAgain);
  1223. }
  1224. #endif // HAVE_WCHAR_H