msb-public
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Code Issues Projects Releases Wiki Activity

compact_enc_det proof of concept

Browse Source
pull/187/head
M66B 4 years ago
parent aeff1a39a0
commit 68bfa6c9a7
29 changed files with 21735 additions and 0 deletions
Show Stats Download Patch File Download Diff File

1
.gitignore vendored
Unescape View File

@ -24,3 +24,4 @@
crowdin.properties
keystore.properties
/app/src/debug/res/xml/providers.xml
/app/.cxx/

1
ATTRIBUTION.md
Unescape View File

@ -28,3 +28,4 @@ FairEmail uses:
* [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE).
* [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE)
* [juniversalchardet](https://github.com/albfernandez/juniversalchardet). Copyright (C) 2001 the Initial Developer. All Rights Reserved. [GNU General Public License Version 2](https://github.com/albfernandez/juniversalchardet#license).
* [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE).

22
app/CMakeLists.txt
Unescape View File

@ -0,0 +1,22 @@
cmake_minimum_required(VERSION 3.4.1)
add_definitions(-DHAVE_MEMRCHR)
add_library( compact_enc_det
SHARED
src/main/jni/charset.cc
src/main/jni/compact_enc_det/compact_enc_det.cc
src/main/jni/compact_enc_det/compact_enc_det_hint_code.cc
src/main/jni/util/encodings/encodings.cc
src/main/jni/util/languages/languages.cc )
include_directories( src/main/jni/ )
find_library( log-lib
log )
target_compile_options(compact_enc_det PRIVATE -Wno-c++11-narrowing)
target_link_libraries( compact_enc_det
${log-lib} )

6
app/build.gradle
Unescape View File

@ -77,6 +77,12 @@ android {
targetCompatibility JavaVersion.VERSION_1_8
}
externalNativeBuild {
cmake {
path "CMakeLists.txt"
}
}
packagingOptions {
exclude 'LICENSES'
exclude 'META-INF/LICENSE.txt'

7
app/src/main/java/eu/faircode/email/CharsetHelper.java
Unescape View File

@ -28,6 +28,12 @@ import java.nio.charset.UnsupportedCharsetException;
class CharsetHelper {
private static final int SAMPLE_SIZE = 1024;
static {
System.loadLibrary("compact_enc_det");
}
private static native String jni_detect(byte[] chars);
static boolean isUTF8(String text) {
// Get extended ASCII characters
byte[] octets = text.getBytes(StandardCharsets.ISO_8859_1);
@ -123,6 +129,7 @@ class CharsetHelper {
static Charset detect(String text) {
try {
byte[] octets = text.getBytes(StandardCharsets.ISO_8859_1);
Log.i("compact_enc_det=" + jni_detect(octets));
int offset = 0;
UniversalDetector detector = new UniversalDetector();

45
app/src/main/jni/charset.cc
Unescape View File

@ -0,0 +1,45 @@
#include <jni.h>
#include <android/log.h>
#include <cstdio>
#include "compact_enc_det/compact_enc_det.h"
void log_android(int prio, const char *fmt, ...) {
if (prio >= ANDROID_LOG_DEBUG) {
char line[1024];
va_list argptr;
va_start(argptr, fmt);
vsprintf(line, fmt, argptr);
__android_log_print(prio, "fairemail.jni", "%s", line);
va_end(argptr);
}
}
extern "C" JNIEXPORT jstring JNICALL
Java_eu_faircode_email_CharsetHelper_jni_1detect(JNIEnv *env, jclass type, jbyteArray _bytes) {
int len = env->GetArrayLength(_bytes);
jbyte *bytes = env->GetByteArrayElements(_bytes, nullptr);
// https://github.com/google/compact_enc_det
bool is_reliable;
int bytes_consumed;
Encoding encoding = CompactEncDet::DetectEncoding(
(const char *) bytes, len,
nullptr, nullptr, nullptr,
UNKNOWN_ENCODING,
UNKNOWN_LANGUAGE,
CompactEncDet::EMAIL_CORPUS,
false,
&bytes_consumed,
&is_reliable);
const char *name = MimeEncodingName(encoding);
log_android(ANDROID_LOG_DEBUG, "detect=%d/%s bytes=%d reliable=%d",
encoding, name, bytes_consumed, is_reliable);
// https://developer.android.com/training/articles/perf-jni#primitive-arrays
env->ReleaseByteArrayElements(_bytes, bytes, JNI_ABORT);
return env->NewStringUTF(name);
}

5719
app/src/main/jni/compact_enc_det/compact_enc_det.cc
View File

File diff suppressed because it is too large Load Diff

83
app/src/main/jni/compact_enc_det/compact_enc_det.h
Unescape View File

@ -0,0 +1,83 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_H_
#define COMPACT_ENC_DET_COMPACT_ENC_DET_H_
#include "util/encodings/encodings.h" // for Encoding
#include "util/languages/languages.h" // for Language
#include <string.h>
namespace CompactEncDet {
// We may want different statistics, depending on whether the text being
// identfied is from the web, from email, etc. This is currently ignored,
// except WEB_CORPUS enables ignoring chars inside tags.
enum TextCorpusType {
WEB_CORPUS,
XML_CORPUS,
QUERY_CORPUS, // Use this for vanilla plaintext
EMAIL_CORPUS,
NUM_CORPA, // always last
};
// Scan raw bytes and detect most likely encoding
// Design goals:
// Skip over big initial stretches of seven-bit ASCII bytes very quickly
// Thread safe
// Works equally well on
// 50-byte queries,
// 5000-byte email and
// 50000-byte web pages
// Length 0 input returns ASCII (aka ISO-8859-1 or Latin1)
//
// Inputs: text and text_length
// web page's url (preferred) or just
// top-level domain name (e.g. "com") or NULL as a hint
// web page's HTTPheader charset= string (e.g. "Latin1") or NULL as a hint
// web page's <meta> tag charset= string (e.g. "utf-8") or NULL as a hint
// an Encoding or UNKNOWN_ENCODING as a hint
// a Language or UNKNOWN_LANGUAGE as a hint
// corpus type from the list above. Currently ignored; may select
// different probability tables in the future
// ignore_7bit if true says to NOT return the pure seven-bit encodings
// ISO-2022-JP (aka JIS), ISO-2022-CN, ISO-2022-KR, HZ, and UTF-7.
// This may save a little scoring time on pure printable ASCII input text
// Outputs: bytes_consumed says how much of text_length was actually examined
// is_reliable set true if the returned encoding is at least 2**10 time more
// probable then the second-best encoding
// Return value: the most likely encoding for the input text
//
// Setting ignore_7bit_mail_encodings effectively turns off detection of
// UTF-7, HZ, and ISO-2022-xx. It is recommended that this flag be true
// when corpus_type is QUERY_CORPUS.
Encoding DetectEncoding(
const char* text, int text_length, const char* url_hint,
const char* http_charset_hint, const char* meta_charset_hint,
const int encoding_hint,
const Language language_hint, // User interface lang
const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
int* bytes_consumed, bool* is_reliable);
// Support functions for unit test program
int BackmapEncodingToRankedEncoding(Encoding enc);
Encoding TopEncodingOfLangHint(const char* name);
Encoding TopEncodingOfTLDHint(const char* name);
Encoding TopEncodingOfCharsetHint(const char* name);
const char* Version(void);
} // End namespace CompactEncDet
#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_H_

54
app/src/main/jni/compact_enc_det/compact_enc_det_fuzz_test.cc
Unescape View File

@ -0,0 +1,54 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#include <stddef.h>
#include <stdlib.h>
#include <memory>
#include "compact_enc_det/compact_enc_det.h"
#include "util/encodings/encodings.h"
#include "util/languages/languages.h"
#include "util/port.h"
#include "gtest/gtest.h"
namespace {
class CompactEncDetFuzzTest : public testing::Test {};
TEST_F(CompactEncDetFuzzTest, TestRandom) {
for (size_t i = 0; i < 16384; ++i) {
unsigned int seed = i;
srand(seed);
size_t length = static_cast<size_t>(rand()) % 1024;
std::unique_ptr<char[]> text(new char[length]);
for (size_t j = 0; j < length; ++j) text[j] = rand();
int bytes_consumed;
bool is_reliable;
CompactEncDet::DetectEncoding(text.get(), length, nullptr, // URL hint
nullptr, // HTTP hint
nullptr, // Meta hint
UNKNOWN_ENCODING,
UNKNOWN_LANGUAGE,
CompactEncDet::WEB_CORPUS,
false, // Include 7-bit encodings?
&bytes_consumed, &is_reliable);
}
}
} // namespace

6326
app/src/main/jni/compact_enc_det/compact_enc_det_generated_tables.h
View File

File diff suppressed because it is too large Load Diff

856
app/src/main/jni/compact_enc_det/compact_enc_det_generated_tables2.h
Unescape View File

@ -0,0 +1,856 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#include "util/basictypes.h"
static const uint8 ced_hires_0[1024] = {
128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
0,135,134,135,89,111,136,129, 81,121,130,40,64,74,78,13, 90,87,103,56,51,37,63,44, 0,97,60,55,143,97,70,100,
29,126,116,149,131,156,108,153, 147,119,135,111,133,135,115,0, 2,13,20,4,12,5,17,5, 4,0,58,50,2,30,80,64,
35,0,0,0,0,22,0,9, 14,12,4,6,7,8,16,3, 147,160,163,141,144,139,146,127, 131,124,23,17,0,11,16,6,
0,68,108,52,129,66,109,65, 113,58,116,124,126,117,88,114, 85,122,102,126,116,122,96,131, 102,126,107,117,49,113,80,128,
0,85,119,107,127,75,102,105, 118,101,108,118,101,117,98,128, 125,114,115,125,115,121,116,121, 125,130,122,119,64,102,78,124,
93,87,64,93,105,92,76,80, 113,82,75,88,89,68,88,97, 87,102,145,90,90,130,106,116, 129,61,81,75,47,69,74,64,
92,98,81,95,81,161,75,81, 101,95,93,75,100,87,108,91, 113,110,118,114,105,105,123,94, 89,92,104,90,62,85,81,84,
47,141,155,88,85,108,121,141, 94,73,82,81,163,125,111,89, 106,133,156,95,80,96,88,61, 65,47,74,55,33,54,59,19,
128,129,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,131,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,128,128,128,135,128,128,128, 128,128,161,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
141,115,118,118,118,128,115,127, 136,136,122,124,125,126,128,121, 147,128,128,128,128,128,128,123, 128,121,128,128,107,154,128,124,
82,141,145,136,128,144,115,123, 124,117,107,104,103,99,107,90, 92,98,102,82,88,134,161,126, 115,104,114,106,73,91,111,84,
128,93,94,94,94,116,91,103, 108,106,98,100,101,102,110,97, 137,133,120,132,146,114,136,131, 144,145,155,131,107,151,144,133,
128,103,106,106,106,128,103,115, 120,118,110,112,113,114,122,109, 157,138,128,117,136,123,136,128, 137,123,152,126,104,136,137,134,
32,94,46,28,98,102,130,97, 59,91,81,73,26,125,102,48, 100,66,105,57,87,121,71,120, 86,64,64,83,68,105,91,70,
32,130,98,103,64,30,123,122, 97,123,58,127,70,74,55,59, 53,99,95,99,105,52,48,73, 52,68,138,89,29,104,97,142,
20,99,54,114,87,86,108,55, 46,27,42,99,130,118,83,95, 138,115,69,67,96,95,125,86, 82,121,92,145,131,142,115,134,
27,53,57,104,139,102,124,124, 135,81,79,148,118,90,71,25, 151,83,146,71,74,129,109,35, 84,115,121,77,48,24,58,63,
29,55,85,51,70,123,21,95, 116,128,90,107,124,74,98,80, 132,125,112,114,118,109,167,103, 57,127,95,84,13,65,84,109,
27,137,123,69,151,81,84,95, 124,80,115,38,140,164,121,122, 86,116,83,102,117,110,111,92, 72,94,114,144,62,90,84,109,
31,128,63,37,14,138,126,79, 132,63,54,95,60,132,89,132, 25,77,134,94,93,154,147,125, 104,98,113,119,70,130,98,111,
24,78,61,30,100,41,65,115, 80,28,93,97,86,38,74,112, 85,105,135,113,132,45,65,30, 126,91,113,114,73,89,65,130,
21,150,114,47,106,125,121,37, 79,158,90,155,73,92,94,107, 87,137,97,135,91,139,152,102, 98,85,126,121,80,140,42,25,
23,47,75,42,64,150,67,93, 100,140,70,62,117,140,54,122, 100,98,95,105,88,100,132,41, 81,137,76,144,60,131,128,66,
23,59,107,144,98,44,20,119, 70,80,58,85,119,93,113,59, 83,28,117,67,135,65,77,60, 130,114,138,111,31,109,92,87,
18,93,79,109,57,128,94,69, 135,54,72,38,53,55,94,56, 133,98,153,147,82,58,124,55, 48,50,153,128,24,54,80,89,
29,130,95,117,49,47,46,91, 164,58,99,148,64,113,86,110, 144,134,49,60,102,62,103,107, 69,89,149,71,113,31,107,71,
25,90,106,113,119,82,75,91, 111,128,139,79,130,101,90,31, 97,103,87,84,53,138,43,87, 155,57,75,150,93,128,120,43,
24,118,82,52,79,56,143,92, 47,103,22,46,70,106,155,129, 93,104,125,42,96,124,137,93, 73,43,134,98,100,111,139,56,
22,65,103,76,38,122,121,155, 111,147,47,76,96,103,153,78, 71,103,137,104,150,89,139,159, 77,120,71,98,98,125,78,143,
};
static const uint8 ced_hires_1[1024] = {
128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
62,124,71,116,117,77,46,71, 78,98,127,132,69,67,72,79, 34,37,88,95,81,88,135,138, 116,123,132,136,97,128,45,101,
68,71,36,0,24,19,3,1, 0,9,60,40,64,146,70,113, 20,16,20,26,13,20,9,12, 14,21,15,19,55,50,99,61,
17,127,133,119,128,134,113,109, 106,124,100,107,119,131,111,122, 138,113,136,154,132,125,120,114, 115,119,101,25,19,31,19,19,
122,112,48,123,115,89,133,132, 124,114,123,130,73,103,139,130, 106,90,103,114,71,97,92,74, 115,112,59,105,83,90,140,121,
118,113,51,133,112,43,123,112, 133,130,102,109,71,110,89,102, 128,125,104,139,115,139,129,134, 112,120,123,117,111,132,129,118,
73,151,165,130,127,146,100,107, 108,121,103,114,156,112,107,111, 125,129,150,135,107,155,118,116, 168,85,79,83,77,89,77,77,
98,101,81,63,71,82,66,64, 60,72,57,62,72,70,60,67, 75,153,138,154,129,161,146,125, 125,139,151,134,139,151,135,146,
59,21,36,18,26,37,21,19, 15,27,12,17,27,25,15,22, 30,34,38,44,31,38,27,30, 32,39,33,37,31,43,31,31,
128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,127,128,124,144,128,127,126, 121,128,118,123,128,128,121,128, 128,128,128,128,128,128,128,128, 128,128,128,128,150,128,128,128,
90,112,109,86,83,124,57,54, 84,70,53,84,62,78,53,75, 124,134,116,95,118,85,129,65, 67,74,68,72,66,78,66,66,
141,134,118,107,135,127,130,134, 118,109,94,99,109,107,97,104, 112,116,120,126,113,120,109,112, 114,121,115,119,113,125,113,113,
161,134,128,112,120,139,115,128, 122,121,106,111,121,119,109,116, 124,128,128,128,125,128,121,124, 126,128,127,128,125,128,125,125,
76,29,158,87,47,122,133,89, 63,101,139,116,147,111,79,122, 60,112,93,68,42,145,73,100, 89,123,46,122,124,112,99,90,
118,56,99,61,145,53,131,144, 66,122,110,97,113,82,19,27, 33,134,120,84,79,124,78,108, 138,87,26,133,114,84,47,165,
62,148,104,137,139,97,125,86, 134,47,127,93,141,85,119,111, 126,56,42,26,102,89,122,48, 59,116,110,106,46,115,64,113,
107,81,49,29,28,64,130,22, 111,53,142,89,89,127,131,86, 122,122,37,143,66,96,82,105, 155,96,154,124,63,121,90,60,
115,107,42,30,134,130,119,108, 66,145,122,85,105,19,88,92, 36,134,34,101,97,112,152,73, 150,89,116,144,71,130,131,103,
88,123,77,30,132,53,98,51, 116,61,95,57,84,109,31,74, 20,98,135,114,78,114,95,123, 45,136,116,111,130,94,120,99,
114,121,64,107,94,104,29,77, 152,133,124,119,135,98,0,78, 118,115,88,102,57,61,51,89, 120,51,115,109,64,111,61,66,
110,135,106,110,98,74,86,137, 124,103,134,80,86,43,76,131, 134,105,91,84,73,34,71,133, 60,111,117,53,90,133,106,61,
147,38,145,28,137,149,110,112, 52,83,99,128,56,129,105,74, 93,60,45,62,73,86,30,82, 80,44,19,97,85,87,108,103,
99,114,94,104,97,104,140,51, 98,62,119,74,85,141,51,23, 71,58,119,110,151,65,116,74, 130,81,86,90,7,99,38,116,
61,59,140,66,91,133,123,152, 80,82,98,67,112,110,110,89, 121,132,120,62,43,74,45,110, 88,146,113,101,134,135,104,131,
110,47,96,67,123,147,69,70, 131,106,98,109,71,118,116,128, 92,100,149,53,144,117,99,136, 127,128,3,43,114,100,85,117,
92,134,156,8,88,62,89,98, 6,19,46,96,145,108,88,69, 78,94,152,96,146,108,133,66, 75,102,101,10,79,44,57,61,
102,96,123,91,109,149,84,63, 39,134,53,110,111,64,97,103, 155,133,105,105,34,125,93,56, 28,84,65,95,103,58,41,83,
48,25,116,120,91,104,124,84, 111,119,105,64,63,97,116,114, 128,103,136,74,59,33,47,93, 54,80,130,39,139,130,119,83,
140,68,94,85,77,79,118,109, 120,106,114,103,115,165,94,79, 60,52,84,77,82,78,49,74, 88,49,48,76,102,99,138,71,
};
static const uint8 ced_hires_2[1024] = {
128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
44,106,18,108,117,63,65,73, 40,57,43,81,77,63,68,90, 54,16,20,95,85,131,112,100, 58,144,140,135,114,96,110,128,
111,93,68,129,135,62,95,32, 82,42,46,10,8,23,10,16, 26,21,70,26,93,84,168,105, 79,71,29,48,15,26,68,128,
28,126,105,123,109,126,102,109, 105,122,72,104,108,132,113,126, 118,82,122,110,129,125,114,134, 113,133,103,54,21,32,21,128,
113,122,134,118,123,77,97,101, 121,135,128,138,125,120,35,120, 31,24,148,129,0,0,0,0, 0,0,0,5,0,0,0,128,
137,137,112,130,105,127,110,121, 86,140,134,139,128,145,17,121, 19,27,60,150,106,62,95,0, 0,0,0,9,0,0,0,128,
86,81,78,87,82,79,82,75, 75,78,69,74,72,87,74,80, 90,85,84,72,92,96,88,82, 98,102,93,112,79,90,79,128,
162,135,144,153,151,137,127,118, 113,125,105,106,91,139,129,109, 124,140,83,71,91,95,87,81, 97,101,92,111,78,89,78,128,
40,35,34,43,36,33,36,29, 29,32,23,28,26,41,28,34, 44,39,38,26,46,50,42,36, 52,56,47,66,33,44,33,128,
128,128,128,131,128,128,130,128, 147,152,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,128,128,138,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
128,128,128,143,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
166,128,128,130,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,146,128,128, 128,128,128,128,148,128,128,128,
139,134,133,75,121,105,114,103, 64,68,129,100,62,76,63,69, 79,74,73,114,137,85,77,71, 174,140,82,101,68,79,68,128,
122,117,114,122,118,115,118,113, 112,117,105,110,108,123,110,116, 126,121,120,108,128,128,124,118, 128,128,128,128,115,126,115,128,
128,128,126,154,128,127,132,123, 137,126,117,122,120,128,122,128, 128,128,128,120,128,128,128,128, 128,128,128,128,127,128,127,128,
67,115,51,135,110,136,135,45, 132,133,58,84,156,90,77,88, 106,108,86,29,51,132,78,133, 122,67,145,151,77,36,57,128,
142,57,81,110,113,64,18,123, 101,136,98,45,102,79,83,102, 85,86,54,106,94,38,117,74, 115,80,89,151,109,98,137,128,
57,126,70,74,47,54,112,60, 146,46,72,36,108,101,48,40, 135,151,135,128,78,111,70,101, 107,100,25,118,65,57,106,128,
67,74,25,123,123,65,42,95, 135,62,82,86,45,70,104,59, 92,30,63,55,143,71,26,83, 97,107,87,111,25,86,45,128,
65,102,90,116,124,16,45,51, 110,129,118,128,73,120,108,139, 139,102,109,110,120,134,85,66, 104,100,96,129,134,94,84,128,
68,135,60,114,78,126,135,56, 22,121,35,117,101,118,119,120, 109,82,99,96,29,91,118,96, 63,104,18,139,47,66,138,128,
78,148,152,94,129,120,107,96, 141,71,52,46,111,50,97,36, 101,133,87,23,91,150,92,38, 110,103,97,84,79,68,85,128,
118,16,110,124,126,74,99,94, 132,79,88,135,103,84,161,153, 35,41,125,78,122,107,74,90, 93,122,134,117,97,91,38,128,
56,121,75,93,68,139,117,78, 23,63,43,75,128,101,116,59, 57,65,121,13,48,108,74,121, 144,120,128,78,95,135,119,128,
141,128,143,68,91,46,137,151, 29,50,32,92,55,73,81,107, 139,145,72,67,54,129,86,35, 105,40,40,99,98,61,142,128,
136,98,109,69,113,126,85,56, 56,103,117,104,17,70,148,128, 55,61,117,55,58,53,121,155, 86,126,75,104,27,105,100,128,
117,110,66,101,135,87,114,88, 69,93,78,106,98,98,117,132, 43,125,54,39,97,119,157,98, 79,131,147,134,59,130,143,128,
60,89,40,119,87,99,73,123, 138,107,135,72,78,97,83,134, 85,110,119,93,43,141,77,98, 119,124,28,104,55,131,118,128,
130,92,71,117,97,53,50,134, 114,136,136,98,51,71,62,115, 96,155,36,1,104,124,59,137, 117,93,26,47,129,107,82,128,
25,44,71,115,56,152,109,40, 117,70,60,119,146,80,83,130, 162,18,126,52,102,137,91,39, 105,113,91,105,37,46,95,128,
68,104,78,51,112,141,87,82, 79,68,59,77,84,49,21,91, 81,68,66,30,149,62,30,70, 81,114,25,107,6,89,62,128,
};
static const uint8 ced_hires_3[1024] = {
145,108,107,112,108,128,119,120, 123,130,108,110,133,138,120,117, 120,120,128,121,121,116,128,119, 128,119,122,114,121,119,123,124,
77,160,155,161,143,37,29,56, 47,35,127,103,75,130,115,110, 149,149,88,80,89,85,145,135, 102,93,107,99,91,89,140,135,
123,90,70,105,58,80,64,65, 54,64,75,114,51,62,65,58, 91,121,130,110,109,105,111,102, 107,100,93,91,94,94,90,92,
173,130,70,57,104,116,65,65, 135,145,70,76,167,129,117,91, 127,129,135,120,115,112,120,109, 117,108,170,131,84,86,91,132,
62,79,115,64,137,72,131,76, 108,65,120,128,125,126,92,124, 97,123,106,137,119,127,102,145, 120,143,106,114,81,121,87,140,
32,95,125,118,134,80,123,115, 112,107,111,122,99,125,101,137, 136,114,118,135,117,125,121,134, 142,145,120,115,95,109,84,135,
149,90,62,97,105,90,90,83, 100,81,70,84,80,69,84,99, 91,95,141,93,85,127,104,122, 139,91,91,84,114,85,87,82,
128,108,87,106,88,166,96,91, 95,101,96,78,98,95,111,100, 124,110,121,124,107,109,128,107, 106,107,102,86,93,92,87,95,
128,128,121,112,126,121,150,106, 125,124,120,96,136,130,135,110, 131,109,148,108,124,113,113,94, 107,139,145,102,114,94,96,95,
92,96,25,73,155,144,172,124, 82,104,144,100,98,115,154,121, 106,60,79,85,106,68,75,81, 150,81,74,87,105,46,52,81,
128,115,75,109,150,190,102,84, 108,126,118,117,98,102,82,70, 101,160,145,120,104,121,112,110, 124,103,124,102,101,99,97,98,
128,121,92,125,93,113,104,105, 94,104,93,99,93,102,105,98, 129,127,166,129,127,135,139,136, 142,143,142,131,131,127,123,126,
128,115,81,140,82,102,107,94, 83,93,83,84,80,91,94,87, 131,132,152,131,139,139,150,144, 142,131,134,133,139,135,137,136,
128,144,99,131,100,120,124,112, 101,111,100,102,98,109,112,105, 126,133,137,137,132,146,148,131, 144,153,139,142,147,123,115,120,
128,132,96,135,97,117,108,109, 98,108,97,99,95,106,109,108, 133,126,134,126,133,142,142,127, 140,133,144,121,134,129,135,117,
128,137,98,138,99,119,110,111, 100,110,99,102,97,112,111,111, 133,122,147,135,132,142,126,127, 144,133,138,140,129,124,135,131,
66,120,131,103,77,97,82,96, 49,122,50,79,108,111,157,61, 103,104,175,93,150,114,129,97, 163,62,107,74,85,83,86,65,
60,104,55,155,93,106,145,111, 154,124,81,79,114,119,104,98, 78,156,105,125,122,58,126,51, 158,67,68,140,110,97,170,83,
57,133,137,102,68,149,108,61, 123,146,85,82,59,65,109,45, 93,71,73,77,93,78,111,93, 84,127,90,164,148,138,91,161,
61,151,94,146,151,131,111,141, 77,115,111,98,134,96,86,120, 72,120,108,84,108,154,91,107, 59,119,71,81,97,99,110,111,
61,123,117,54,44,130,154,70, 125,120,23,146,110,91,116,88, 138,83,134,93,150,98,76,53, 90,99,140,60,102,90,84,118,
48,50,50,114,97,154,78,31, 88,101,72,126,31,104,99,105, 108,142,96,135,96,117,112,62, 90,103,116,59,144,151,100,63,
60,111,72,74,81,149,104,72, 154,140,90,149,106,117,107,164, 94,68,40,103,113,75,95,119, 78,119,86,84,153,133,135,61,
57,55,162,113,44,109,106,106, 156,45,51,86,91,122,91,49, 59,115,122,101,140,123,148,88, 128,112,113,99,106,160,94,146,
60,103,65,134,55,108,82,107, 109,74,62,66,51,60,154,112, 107,132,118,98,153,90,145,96, 127,103,138,131,32,128,125,71,
83,37,62,74,156,128,140,85, 67,149,49,158,98,78,96,58, 74,105,154,86,85,104,70,106, 71,139,144,112,70,87,62,57,
61,52,121,154,38,91,135,69, 50,69,82,123,119,105,113,111, 94,87,99,91,73,100,97,82, 110,100,136,69,126,142,55,51,
74,69,108,46,132,141,110,157, 133,69,143,44,100,133,144,154, 148,71,78,118,106,110,142,141, 77,145,112,124,118,87,69,36,
55,94,65,106,116,57,117,82, 111,82,114,127,58,134,115,141, 151,122,124,60,133,57,158,91, 139,85,137,75,157,84,128,56,
59,132,91,110,40,123,69,72, 156,84,95,148,88,151,121,95, 59,132,143,80,88,128,94,95, 76,120,121,149,98,86,94,91,
63,124,105,95,61,107,107,129, 65,159,114,147,62,170,118,134, 140,104,135,145,113,123,129,67, 107,117,135,114,64,66,85,147,
65,101,84,63,70,103,113,106, 138,80,159,36,114,114,86,133, 93,102,95,90,159,137,73,65, 75,126,75,50,154,87,101,119,
};
static const uint8 ced_hires_4[1024] = {
125,128,119,118,134,121,118,117, 121,123,122,116,128,124,115,120, 123,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,133,
89,125,49,76,12,21,28,18, 21,36,39,85,85,24,34,27, 45,91,39,57,98,41,58,61, 34,65,57,69,88,91,63,51,
88,107,83,89,76,130,128,123, 123,120,114,104,121,106,102,107, 111,122,121,124,104,124,105,123, 75,193,173,183,166,161,151,140,
57,123,96,99,90,98,90,87, 92,100,90,86,102,110,91,100, 111,104,117,127,115,104,104,107, 93,107,76,126,75,129,66,99,
132,140,43,135,109,85,142,143, 142,123,145,142,88,114,151,140, 111,113,116,119,86,108,108,99, 125,132,66,121,102,103,162,139,
127,140,45,144,105,38,131,121, 150,137,124,119,85,120,100,111, 132,147,116,143,129,149,144,158, 121,139,129,132,129,144,150,135,
118,171,152,134,113,134,101,109, 118,121,117,117,163,115,111,113, 122,144,155,132,114,158,126,133, 170,108,78,105,109,94,91,87,
107,128,102,85,100,86,92,85, 99,107,95,79,99,102,94,112, 103,175,150,158,143,171,161,149, 134,158,157,149,157,163,156,163,
109,134,92,99,79,88,99,96, 113,116,109,96,117,109,94,107, 109,117,140,123,112,109,118,129, 82,99,110,93,90,120,93,105,
174,130,49,65,96,52,55,98, 100,54,66,43,94,53,51,88, 117,70,89,96,67,74,69,96, 111,61,42,98,52,58,55,82,
104,127,95,100,95,101,95,103, 111,102,126,112,114,120,101,120, 111,128,128,120,116,125,119,128, 87,104,85,98,95,101,98,94,
127,145,120,137,126,116,135,123, 128,134,149,145,134,137,137,135, 138,172,148,150,144,142,146,155, 116,128,113,126,123,128,126,122,
127,149,116,132,127,126,132,129, 142,132,146,137,136,137,141,133, 138,152,166,158,154,155,150,158, 104,121,102,120,112,119,115,111,
147,147,113,127,121,115,115,109, 131,129,132,124,135,125,120,129, 126,150,153,163,145,131,145,157, 122,128,120,128,128,128,128,128,
132,146,128,118,123,118,136,128, 140,142,131,133,128,133,123,134, 135,144,155,144,129,141,146,152, 119,128,117,129,127,128,128,126,
136,161,124,119,112,127,127,117, 125,135,136,116,159,139,127,131, 135,144,151,151,142,140,146,158, 121,128,119,128,128,128,128,128,
56,117,131,52,109,89,49,75, 81,137,58,134,87,143,104,64, 67,158,68,113,116,101,108,155, 107,161,107,92,127,123,72,117,
86,78,40,98,54,51,93,94, 146,77,132,88,105,47,86,121, 62,41,132,68,70,140,41,104, 135,103,110,99,123,62,88,138,
101,108,91,108,121,139,137,42, 81,139,135,123,107,124,135,85, 101,65,99,90,94,85,106,115, 139,139,64,101,103,134,95,143,
70,126,127,75,75,101,140,147, 87,164,107,104,165,99,81,128, 127,71,73,92,129,93,149,99, 116,117,63,116,99,109,92,116,
76,80,61,56,75,111,73,103, 101,107,139,145,119,80,146,97, 80,85,92,153,99,90,106,97, 63,145,46,83,77,87,100,104,
146,115,124,137,168,42,104,127, 143,56,58,87,91,127,81,104, 84,80,65,78,61,37,98,132, 153,100,149,107,103,115,86,77,
130,158,89,75,83,106,85,85, 154,103,67,119,146,96,127,133, 57,111,108,142,164,78,117,83, 96,129,56,73,114,66,45,67,
144,99,131,129,98,141,102,131, 94,130,123,75,85,40,91,109, 73,161,102,81,53,102,161,124, 69,88,72,127,111,144,48,90,
63,71,74,140,135,125,89,112, 80,139,117,97,84,70,104,105, 134,91,113,99,91,128,126,100, 80,119,115,145,82,39,81,166,
114,92,92,84,94,122,49,104, 105,137,124,110,127,85,72,101, 21,83,117,52,78,79,102,69, 159,137,104,143,160,134,95,104,
121,128,79,153,100,154,100,103, 107,108,78,130,91,172,136,158, 103,65,53,145,51,100,86,107, 108,103,136,103,149,108,105,77,
79,64,42,108,78,77,129,65, 55,77,109,88,67,93,68,52, 53,94,114,121,109,121,106,60, 146,113,83,116,96,136,58,100,
84,132,50,139,86,86,148,141, 107,91,140,68,130,131,107,42, 102,135,166,160,54,62,91,126, 79,136,52,161,124,123,100,71,
70,99,52,78,44,116,54,121, 67,112,82,59,164,83,92,133, 108,81,120,156,91,84,129,131, 107,110,139,91,122,128,82,92,
118,77,72,119,66,133,131,40, 125,123,54,65,72,159,54,65, 84,86,68,142,68,101,155,79, 102,155,56,134,117,163,131,155,
66,82,99,78,48,100,154,96, 81,163,89,132,131,155,126,110, 54,98,82,94,51,152,112,120, 138,77,149,127,74,99,109,80,
};
static const uint8 ced_hires_5[1024] = {
128,127,124,128,128,147,138,139, 162,124,125,126,127,127,152,184, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
59,56,52,84,74,62,105,50, 75,46,54,59,89,56,113,130, 119,132,110,115,136,127,123,76, 85,121,121,94,112,104,106,128,
126,117,109,74,77,123,124,119, 122,105,96,86,86,78,87,79, 94,169,180,158,166,149,129,116, 113,117,109,100,100,85,105,128,
59,98,77,92,84,100,75,88, 93,94,54,78,89,98,98,106, 103,65,103,95,118,105,93,113, 76,102,66,54,135,52,60,128,
114,126,138,119,130,83,102,112, 141,139,142,144,138,118,52,132, 48,39,161,146,26,1,0,0, 0,6,0,43,34,8,25,128,
137,140,115,130,111,132,114,131, 105,143,147,144,140,142,33,132, 35,41,72,166,126,73,105,0, 0,0,0,53,3,50,23,128,
112,128,104,79,120,147,178,100, 105,85,90,74,80,77,120,87, 130,112,180,83,120,100,91,85, 85,95,84,102,99,90,92,128,
162,138,147,153,157,142,131,128, 132,128,118,111,103,136,145,120, 140,154,95,87,111,106,97,91, 91,101,86,97,105,96,98,128,
96,79,91,123,95,99,109,128, 124,80,77,74,81,79,85,86, 101,94,91,83,107,102,93,87, 87,97,91,93,101,92,94,128,
130,76,66,68,64,65,77,111, 81,61,51,63,45,76,57,136, 63,56,53,46,77,64,62,49, 49,59,123,97,63,54,56,128,
86,86,81,86,91,131,96,85, 96,81,82,79,84,117,90,91, 106,99,96,88,112,107,98,92, 95,102,87,98,113,97,99,128,
114,112,109,114,116,119,125,113, 127,111,110,121,112,112,118,119, 128,127,124,116,128,128,126,120, 120,128,115,126,140,125,127,128,
103,101,98,103,105,118,105,102, 113,98,99,96,101,101,107,108, 147,116,113,105,128,124,115,109, 109,119,104,115,136,114,116,128,
121,119,116,121,123,142,128,128, 128,116,117,114,119,119,125,126, 128,128,128,123,128,128,128,127, 127,128,122,128,128,128,128,128,
118,116,113,118,120,127,150,122, 139,118,114,111,116,116,122,123, 128,128,128,120,128,128,128,124, 124,128,119,128,128,128,128,128,
127,118,115,120,139,136,123,124, 134,127,116,113,118,118,124,125, 128,128,128,122,128,128,128,126, 126,128,121,128,128,128,128,128,
144,113,79,134,119,150,168,110, 109,123,87,135,164,70,123,153, 60,128,94,93,119,96,90,82, 80,91,52,120,157,65,95,128,
152,77,95,138,152,63,122,108, 72,111,161,80,53,163,71,71, 154,47,85,88,86,124,118,33, 115,126,72,98,91,95,82,128,
128,146,140,137,59,123,98,75, 133,154,41,39,133,67,133,64, 115,98,30,37,84,76,58,26, 78,73,153,84,82,120,63,128,
106,93,89,71,142,126,122,109, 127,113,101,74,47,71,102,125, 99,87,54,87,115,144,165,81, 99,49,47,81,86,139,114,128,
95,47,91,59,105,149,149,123, 89,57,69,124,88,133,119,156, 141,64,149,165,107,86,69,108, 135,57,146,122,119,128,82,128,
44,75,63,157,128,38,92,158, 41,77,129,36,81,96,88,54, 44,93,58,46,115,99,80,131, 96,65,76,87,70,72,83,128,
165,111,57,90,104,46,83,56, 72,75,68,75,96,118,139,55, 81,118,70,56,99,104,83,106, 143,142,105,139,81,83,155,128,
86,127,119,93,103,124,116,150, 118,69,88,93,98,72,109,119, 122,130,119,64,126,86,111,57, 87,131,74,125,103,69,156,128,
98,68,93,126,62,132,161,123, 149,72,93,84,65,65,112,144, 101,154,60,90,129,83,167,144, 140,140,113,105,164,79,61,128,
112,113,139,157,96,91,153,68, 106,113,83,60,100,103,74,71, 111,102,68,110,41,73,115,67, 93,107,158,143,83,149,120,128,
95,117,116,117,99,81,87,114, 115,118,121,91,142,90,110,101, 82,114,128,148,135,125,105,70, 80,91,110,89,129,99,140,128,
85,169,69,133,73,49,110,60, 96,123,99,77,114,72,157,121, 140,133,145,94,106,142,90,139, 96,154,153,76,95,136,38,128,
144,125,98,54,162,77,113,95, 108,88,42,67,143,69,97,30, 68,71,136,64,127,135,138,72, 107,115,59,137,148,97,158,128,
122,156,149,122,108,71,79,147, 128,144,38,68,131,98,99,121, 164,143,123,52,140,126,130,91, 154,67,95,124,146,43,91,128,
124,62,79,123,127,91,91,135, 121,69,95,76,76,128,74,47, 52,71,100,47,105,141,145,88, 123,140,106,86,152,110,91,128,
123,121,138,108,94,76,62,115, 57,128,70,49,149,129,153,78, 71,120,117,115,74,80,143,95, 64,35,73,80,92,79,49,128,
};
static const uint8 ced_hires_6[1024] = {
131,89,116,77,83,97,102,136, 133,105,111,105,87,87,87,105, 108,118,117,124,140,105,110,104, 68,96,93,87,81,101,115,102,
17,136,152,147,132,68,100,48, 56,48,81,53,20,72,87,102, 143,155,79,74,108,84,137,154, 76,94,89,93,83,102,92,87,
66,79,73,41,49,115,104,62, 46,89,89,70,100,49,89,121, 91,76,85,75,125,76,65,130, 119,156,144,96,123,158,156,118,
20,106,67,42,24,96,44,57, 144,132,70,77,156,109,89,83, 121,135,126,114,134,111,112,128, 91,109,152,125,78,102,101,136,
43,96,88,68,104,16,25,125, 72,107,35,59,22,13,0,14, 89,145,128,73,85,131,89,165, 108,108,98,155,106,124,167,120,
128,81,99,82,73,82,85,63, 69,94,110,62,46,48,43,56, 155,167,150,139,160,127,125,138, 97,126,85,77,73,94,89,89,
67,136,152,100,83,91,124,114, 106,89,92,124,146,131,103,110, 120,137,118,86,122,92,116,123, 90,103,95,79,104,103,104,97,
122,74,119,104,124,104,96,98, 90,118,93,81,76,100,118,130, 143,123,120,167,152,126,138,151, 125,120,83,101,102,118,116,108,
79,88,103,83,60,34,70,38, 92,77,94,49,116,95,82,88, 43,144,139,131,133,98,98,121, 86,94,95,101,96,80,109,97,
88,98,105,102,82,110,51,104, 115,100,128,93,119,80,46,69, 110,117,74,90,120,121,93,128, 119,85,83,79,94,103,104,110,
89,75,115,84,121,92,114,102, 122,90,122,128,106,108,84,118, 107,133,113,126,141,123,115,158, 103,135,99,112,107,138,121,141,
94,79,115,97,115,104,113,108, 122,102,110,119,95,109,90,121, 119,127,118,122,154,126,117,158, 112,137,104,112,113,135,125,129,
121,83,109,77,84,83,62,98, 104,84,113,116,93,81,73,85, 118,113,99,96,123,96,108,131, 95,111,79,75,101,113,113,103,
140,66,86,79,79,90,80,94, 100,88,90,93,77,79,74,87, 118,152,103,106,142,108,110,143, 111,131,122,106,115,147,126,135,
166,67,85,78,78,89,79,93, 99,87,89,92,76,78,73,86, 130,127,102,124,155,105,121,151, 123,149,123,109,120,162,148,136,
154,75,95,88,88,99,89,103, 109,97,99,102,86,88,83,96, 149,130,112,108,140,112,120,145, 114,125,119,102,115,145,139,135,
31,150,134,141,48,120,36,27, 151,116,122,82,89,136,91,28, 119,68,44,150,145,40,107,55, 36,92,23,66,46,41,24,59,
41,45,12,0,141,58,83,30, 9,9,61,106,0,45,9,16, 14,23,66,152,39,5,42,56, 144,155,123,64,151,84,75,33,
107,62,98,81,89,126,87,70, 167,111,120,112,104,100,108,129, 72,124,176,74,96,72,84,77, 115,63,29,72,108,97,111,154,
85,144,123,65,80,95,99,66, 65,55,176,121,104,135,27,146, 62,44,158,114,101,106,105,126, 60,80,72,160,55,126,113,101,
36,0,87,20,72,44,8,0, 67,127,87,121,10,84,77,51, 34,21,68,32,57,64,38,24, 9,14,132,0,17,87,36,41,
67,104,95,43,73,154,82,100, 126,72,76,68,26,71,58,32, 24,45,36,21,55,166,135,135, 134,126,7,36,81,101,84,170,
75,82,59,49,43,129,38,169, 67,116,63,94,69,64,52,101, 133,117,129,120,66,52,28,69, 66,82,50,119,104,83,74,71,
47,131,113,126,57,142,116,109, 99,107,76,126,9,6,123,152, 96,136,103,126,142,67,90,109, 102,148,97,101,76,87,83,95,
49,8,75,129,53,138,158,150, 98,83,101,27,3,66,159,125, 128,114,158,134,119,126,153,139, 137,128,35,126,84,61,108,84,
34,0,18,21,24,22,93,33, 38,1,18,168,88,79,156,105, 149,47,24,56,50,69,76,64, 20,102,71,68,46,62,48,58,
30,53,84,128,83,135,48,120, 32,36,94,29,44,92,100,149, 138,79,46,24,153,61,30,49, 158,143,56,138,115,118,110,93,
74,93,20,31,73,44,42,111, 52,105,68,74,46,43,29,66, 45,67,37,7,76,88,82,101, 52,70,30,45,51,78,66,100,
54,32,43,64,72,117,82,71, 69,29,37,29,4,167,126,75, 26,161,51,143,53,30,104,121, 74,79,144,27,157,129,153,117,
55,53,53,87,24,19,80,75, 68,63,40,42,111,20,59,68, 72,114,50,26,138,29,70,86, 43,51,156,34,99,130,66,103,
53,0,46,31,71,79,87,57, 59,51,34,39,24,0,25,66, 1,47,140,45,144,92,34,50, 90,75,50,2,24,40,147,114,
63,150,105,113,95,108,52,72, 62,157,153,73,141,125,32,38, 104,104,25,64,137,147,31,105, 16,139,27,46,1,90,101,23,
};
static const uint8 ced_hires_7[1024] = {
120,127,110,107,125,119,124,105, 127,128,122,128,102,123,102,99, 129,154,113,122,107,119,106,109, 116,98,104,108,120,117,116,111,
91,83,50,46,97,86,122,78, 99,143,61,80,78,80,62,52, 17,15,56,62,51,91,64,93, 138,118,150,133,136,119,114,121,
173,162,166,159,148,141,205,123, 204,166,143,146,120,144,133,153, 134,173,138,140,103,106,108,117, 108,108,71,103,148,151,114,164,
86,147,121,122,141,130,130,109, 132,142,124,133,108,143,112,113, 130,102,125,146,122,120,113,107, 113,92,86,127,123,136,75,107,
137,141,85,124,139,117,115,121, 96,81,80,124,153,77,61,54, 173,171,86,199,83,61,48,51, 58,40,46,50,62,59,58,53,
145,149,144,119,151,127,147,125, 167,161,116,130,99,120,128,128, 128,103,131,112,106,147,108,130, 151,83,89,93,105,102,101,96,
129,134,99,103,101,147,132,118, 111,96,87,94,100,88,67,64, 93,74,78,87,72,84,86,74, 81,63,69,73,85,82,81,76,
125,123,130,99,124,109,144,109, 111,115,106,113,93,107,93,83, 92,101,134,122,117,122,132,121, 100,122,88,92,104,101,112,95,
117,129,112,84,103,101,127,90, 87,91,82,100,75,179,112,185, 110,132,107,115,106,185,103,103, 136,103,135,134,129,86,137,125,
139,147,101,95,125,132,138,134, 114,118,109,116,89,113,94,123, 98,96,100,109,94,106,93,96, 103,85,91,95,107,104,103,98,
159,165,129,148,166,141,172,150, 179,163,171,181,122,149,165,145, 138,128,145,149,124,146,132,127, 143,122,121,133,144,135,162,148,
171,165,133,151,163,146,162,148, 172,168,156,165,126,154,137,132, 143,140,143,155,135,156,140,142, 142,127,130,137,149,145,153,146,
132,133,96,93,127,105,110,92, 113,117,108,115,88,109,88,85, 94,169,171,176,140,166,165,125, 134,143,158,156,158,166,150,158,
169,154,126,147,134,128,175,148, 138,128,128,128,118,128,118,115, 124,125,128,128,123,128,122,125, 128,114,120,124,128,128,128,127,
160,159,125,129,155,136,156,153, 149,128,128,128,117,128,117,114, 123,124,128,128,122,128,121,124, 128,113,119,123,128,128,128,126,
168,147,128,128,128,139,128,143, 141,128,128,128,127,128,127,124, 128,128,128,128,128,128,128,128, 128,123,128,128,128,128,128,128,
26,71,23,0,18,145,93,129, 88,129,0,160,78,145,20,29, 21,64,41,28,155,90,68,73, 46,65,128,9,130,142,83,131,
59,84,81,102,49,55,8,132, 69,48,0,80,0,123,0,36, 37,33,0,62,129,121,64,162, 123,122,0,154,69,160,143,89,
155,85,24,119,133,127,86,100, 126,180,98,119,16,37,81,98, 82,71,113,46,31,50,30,33, 70,148,98,135,141,102,162,99,
94,74,90,98,122,40,42,20, 72,125,152,93,42,97,112,5, 103,130,75,82,21,65,120,149, 107,58,80,48,61,150,41,63,
118,82,161,115,70,0,55,0, 13,151,100,56,59,0,25,151, 102,102,94,0,133,95,68,98, 35,166,95,80,163,108,138,85,
47,27,46,3,18,108,113,138, 156,140,109,142,22,49,123,83, 98,107,78,74,32,109,58,60, 83,0,109,0,1,0,105,55,
71,49,44,31,49,30,38,141, 120,37,96,114,14,14,14,21, 22,111,109,31,70,103,7,100, 17,109,62,65,72,49,18,128,
40,154,151,137,126,105,104,61, 119,144,127,42,0,0,164,141, 135,126,95,119,107,112,29,0, 0,0,99,11,15,13,9,0,
108,133,116,95,53,149,107,100, 56,66,115,79,15,100,65,0, 1,0,7,140,111,132,101,0, 95,4,92,92,46,22,161,72,
58,38,113,14,88,36,11,97, 67,63,27,39,143,108,133,82, 135,0,52,60,72,93,90,149, 109,147,143,103,45,159,148,153,
132,115,1,88,45,51,72,56, 44,62,5,83,23,28,153,121, 167,37,147,101,0,82,26,76, 97,99,17,1,1,0,40,86,
83,46,30,15,41,21,26,83, 78,66,41,84,84,78,0,41, 50,103,63,122,73,69,0,61, 48,0,97,105,70,96,62,90,
85,84,93,6,54,125,46,123, 102,62,7,71,96,44,54,0, 54,4,166,155,31,144,107,0, 87,55,65,143,79,47,0,5,
163,95,121,171,161,163,63,142, 100,152,141,64,95,125,105,56, 0,89,99,83,64,43,95,89, 83,50,23,0,68,0,0,76,
92,65,42,71,88,88,176,120, 171,105,154,154,18,44,54,106, 112,92,134,139,0,134,133,130, 116,101,84,51,64,46,90,155,
152,122,117,113,19,86,30,51, 85,122,99,90,18,146,45,102, 58,13,43,75,104,119,101,0, 77,0,0,0,146,0,119,0,
};
static const uint8 ced_hires_8[1024] = {
130,127,155,168,146,193,173,178, 138,176,189,169,199,152,146,172, 128,127,128,122,128,128,109,122, 115,128,126,128,120,128,128,128,
142,160,113,159,112,138,145,87, 133,125,100,110,109,101,70,134, 52,81,111,42,63,48,9,22, 34,44,34,64,45,60,106,128,
98,91,146,105,104,134,59,53, 74,82,71,49,70,51,52,76, 77,69,81,64,94,79,51,64, 57,75,68,77,62,74,106,128,
90,132,91,134,101,123,97,99, 109,125,92,97,124,114,109,140, 123,84,137,120,149,126,95,134, 90,124,96,82,140,78,116,128,
68,69,88,133,99,128,124,111, 101,105,137,125,134,111,84,138, 137,75,81,64,111,89,60,64, 57,75,68,77,62,215,110,128,
111,181,158,179,148,178,161,155, 155,149,184,172,200,153,139,187, 156,138,159,137,153,154,108,115, 129,118,111,120,105,117,128,128,
91,101,111,146,104,81,82,70, 84,86,94,72,93,74,75,99, 100,92,104,87,117,102,74,87, 80,98,91,100,85,97,128,128,
137,111,137,155,136,166,170,156, 138,140,168,155,174,149,139,169, 119,111,123,106,128,121,93,106, 99,117,110,119,104,116,128,128,
127,121,128,160,166,147,149,173, 185,181,184,160,174,149,143,166, 164,138,146,126,151,136,178,115, 117,182,167,95,80,92,124,128,
113,114,134,159,120,170,172,178, 160,151,180,174,185,158,134,167, 122,114,126,109,128,124,181,188, 193,130,119,145,158,155,152,128,
155,164,146,164,148,177,155,147, 159,166,174,167,174,147,135,174, 105,97,186,168,122,107,79,92, 85,103,96,105,90,102,128,128,
161,164,146,176,146,180,170,159, 156,165,177,172,179,155,145,171, 111,103,162,180,176,113,104,98, 91,109,102,111,96,108,128,128,
187,159,158,190,172,180,181,159, 155,156,183,160,177,165,170,171, 148,172,125,108,128,141,95,108, 101,119,112,121,126,118,128,128,
128,128,120,133,128,128,128,121, 128,128,128,123,128,125,126,128, 128,128,128,128,128,128,125,128, 166,128,128,128,131,128,128,128,
128,128,119,128,128,128,128,122, 129,131,128,122,128,124,125,128, 128,128,128,128,128,128,124,128, 128,128,128,128,128,128,128,128,
128,128,128,154,128,128,133,128, 142,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128, 128,128,128,128,128,128,128,128,
20,141,64,92,31,78,151,53, 145,20,53,48,65,154,102,114, 93,115,27,0,49,93,58,115, 144,65,153,88,149,88,17,128,
84,40,161,60,108,42,125,33, 139,68,83,54,43,81,132,82, 69,84,108,0,122,85,6,2, 53,70,102,0,36,34,41,128,
100,104,66,145,69,110,103,89, 76,65,98,98,84,63,121,133, 106,95,152,44,168,45,122,127, 129,38,117,149,53,137,128,128,
134,104,149,56,114,62,74,104, 71,45,50,137,132,122,110,44, 112,56,54,136,150,128,59,0, 0,81,103,3,0,0,33,128,
81,17,22,151,126,101,49,126, 22,36,76,149,81,76,40,83, 64,111,53,68,65,145,96,0, 110,0,87,0,39,98,99,128,
104,20,78,59,26,158,114,116, 105,158,62,77,97,99,149,60, 158,104,111,51,111,77,41,70, 35,124,61,140,87,83,139,128,
89,103,71,128,99,85,122,108, 62,50,67,61,120,64,44,79, 43,9,107,179,176,163,107,154, 141,87,110,163,2,37,115,128,
0,162,20,49,20,47,97,121, 95,83,96,88,115,58,47,97, 61,11,73,19,29,17,0,0, 0,148,82,94,111,89,40,128,
120,92,69,58,56,38,49,120, 38,153,95,56,69,140,92,44, 157,155,88,91,145,0,89,50, 109,97,0,66,0,18,97,128,
114,13,57,120,93,77,153,77, 152,135,99,80,88,24,69,60, 107,53,43,2,13,0,131,78, 141,49,121,0,119,153,135,128,
0,4,6,55,124,56,65,29, 21,40,149,41,100,137,17,39, 3,163,111,111,140,0,52,59, 72,113,76,118,116,81,92,128,
47,32,26,62,0,51,45,166, 142,38,168,49,140,20,96,145, 124,74,90,169,73,151,143,89, 64,116,26,75,20,162,143,128,
0,0,91,31,9,39,39,26, 90,29,42,35,53,27,112,56, 58,67,4,76,79,51,153,115, 122,50,137,120,19,114,123,128,
78,120,61,100,65,59,85,52, 72,57,93,54,61,10,72,78, 56,75,77,0,33,34,43,38, 0,0,14,44,0,44,24,128,
151,95,67,85,59,72,70,135, 53,60,105,54,54,25,160,139, 142,61,116,126,90,1,104,159, 139,115,141,100,41,62,100,128,
0,0,16,46,152,103,55,43, 56,47,60,148,157,92,133,148, 0,0,158,50,138,103,111,67, 150,157,47,12,78,21,146,128,
};
static const uint8 ced_hires_9[1024] = {
130,71,129,141,137,60,133,127, 113,78,80,127,94,127,117,51, 103,62,119,129,148,68,133,101, 38,42,158,125,112,190,163,124,
139,45,127,155,132,105,145,125, 136,78,127,128,145,136,141,61, 122,142,128,129,137,97,140,143, 1,3,145,120,131,129,135,62,
124,125,104,88,80,128,69,72, 68,127,36,103,135,92,101,131, 77,126,132,112,51,143,88,87, 94,151,72,95,120,110,80,161,
81,123,101,98,60,133,79,54, 52,155,28,125,46,72,67,101, 90,70,86,76,45,128,37,119, 86,129,81,86,102,49,41,88,
96,132,92,99,114,136,90,108, 121,130,15,114,132,93,122,129, 135,112,129,111,82,133,131,124, 119,129,105,108,112,39,105,147,
115,68,124,132,149,103,109,139, 127,72,144,126,137,143,146,76, 122,99,138,142,133,69,134,126, 32,11,131,145,107,148,126,61,
77,126,72,78,49,124,127,85, 75,135,43,82,121,79,83,143, 74,43,129,98,111,130,44,45, 87,106,101,92,138,51,52,89,
95,126,67,51,135,116,72,112, 51,131,24,106,133,97,111,143, 71,33,136,82,68,135,62,78, 22,49,63,84,114,36,104,82,
46,129,63,71,69,93,73,121, 119,140,58,72,114,107,136,142, 96,57,126,112,97,126,48,117, 55,54,76,101,134,53,81,140,
139,93,114,144,122,123,128,130, 146,120,146,137,129,138,134,101, 109,151,120,136,133,78,129,131, 0,6,154,131,126,122,151,58,
103,57,111,129,142,74,122,99, 101,56,40,136,131,124,138,107, 93,87,91,150,153,48,74,96, 29,52,125,155,90,119,156,95,
71,137,72,129,58,106,76,87, 65,135,10,79,127,82,98,139, 95,29,127,121,126,138,126,104, 58,30,84,95,111,27,55,91,
148,128,84,57,100,136,84,113, 74,138,28,114,116,81,111,129, 86,138,17,107,80,134,130,64, 160,116,80,94,104,93,101,84,
76,131,112,98,55,135,114,120, 76,132,40,106,110,135,121,131, 146,114,77,99,59,136,66,90, 106,140,89,78,129,75,105,92,
119,136,75,128,125,129,141,113, 81,139,14,124,57,64,124,132, 74,124,75,119,117,122,73,70, 119,145,101,93,142,120,105,136,
111,62,156,130,141,107,139,152, 124,103,148,125,129,141,131,71, 138,116,130,137,129,82,139,148, 0,0,125,133,142,126,134,55,
72,116,48,77,48,122,93,65, 57,115,22,108,127,51,83,142, 87,100,151,86,87,131,39,43, 86,113,70,90,98,53,85,72,
149,20,138,146,143,112,62,118, 158,65,125,130,127,150,129,33, 114,116,113,124,140,58,141,141, 24,20,165,129,84,163,122,98,
106,140,115,104,111,132,132,124, 123,125,23,112,87,134,120,128, 118,121,93,124,114,141,143,116, 117,135,106,137,107,90,103,86,
104,114,115,105,103,116,131,80, 117,119,30,147,132,120,108,122, 147,136,106,132,147,127,81,123, 134,114,71,127,123,66,120,147,
106,131,95,77,65,135,100,63, 69,133,8,113,86,81,116,133, 74,114,131,122,67,128,53,135, 151,132,60,61,111,92,110,88,
161,88,152,116,143,105,115,146, 136,61,114,137,128,126,120,44, 151,90,132,138,125,54,155,126, 8,28,138,144,121,160,160,75,
87,124,98,82,154,145,88,92, 46,138,32,132,81,107,138,90, 81,34,84,103,27,129,104,81, 109,61,73,49,116,47,121,91,
72,136,72,95,97,132,49,95, 111,125,35,117,129,103,124,133, 112,95,117,134,95,115,67,97, 102,148,126,135,108,97,99,123,
165,12,134,132,98,115,126,113, 92,122,36,138,34,110,157,63, 107,139,40,148,113,22,79,71, 31,22,127,133,78,127,95,90,
38,12,112,64,133,138,43,108, 175,16,168,125,147,148,91,8, 100,54,87,106,121,17,106,130, 17,47,101,161,65,95,131,83,
119,144,110,98,143,110,64,120, 43,121,32,102,111,138,133,123, 86,119,120,80,52,126,109,143, 98,130,90,84,85,38,91,148,
95,124,58,99,38,144,76,50, 77,139,32,143,125,91,120,110, 106,28,108,77,132,126,36,92, 136,57,38,62,101,62,40,90,
47,23,65,73,87,20,162,94, 103,127,100,162,133,118,116,37, 117,60,94,98,151,51,49,98, 24,20,116,121,117,51,42,90,
54,123,33,45,31,151,56,33, 42,145,36,29,25,35,119,81, 32,43,48,22,12,118,44,43, 149,32,38,50,76,65,37,95,
46,130,41,34,22,145,59,49, 59,134,20,125,111,79,130,70, 69,37,81,47,140,127,73,75, 123,43,61,147,80,46,72,78,
115,32,81,74,58,149,102,67, 71,46,65,60,49,57,61,30, 68,174,54,51,45,52,72,53, 57,53,71,79,105,84,67,123,
};
static const uint8 ced_hires_10[1024] = {
58,122,143,128,136,100,142,138, 75,129,136,141,139,140,49,124, 126,129,133,99,142,134,152,137, 120,138,45,14,25,143,136,140,
125,108,87,90,85,127,88,79, 128,11,95,136,93,104,132,81, 132,106,62,140,86,72,90,83, 121,114,149,146,89,121,124,118,
133,75,100,96,92,129,79,127, 127,44,117,130,100,119,132,110, 114,131,93,115,52,110,92,107, 134,108,137,144,103,119,70,100,
129,70,93,99,140,110,67,56, 133,40,103,132,85,100,142,72, 134,84,77,132,70,53,62,94, 79,31,106,60,43,131,68,32,
132,100,125,106,118,135,136,106, 130,30,113,134,97,123,128,124, 125,111,84,130,80,106,89,109, 111,56,133,127,111,123,95,111,
67,126,128,140,149,99,132,133, 69,144,128,139,143,147,68,121, 136,137,133,67,119,131,136,128, 136,149,79,10,23,96,114,96,
125,102,73,98,152,143,106,80, 137,19,128,84,119,136,89,83, 74,107,89,132,74,58,81,126, 52,49,110,73,108,116,88,42,
142,113,137,119,142,109,104,86, 122,38,105,107,137,130,124,108, 116,84,69,126,79,79,97,100, 101,51,133,133,97,103,115,109,
96,116,125,136,124,122,129,150, 119,148,137,131,138,134,97,111, 120,136,131,74,132,148,147,147, 126,125,57,11,16,101,139,150,
64,113,99,105,134,74,84,122, 61,63,130,131,122,139,112,93, 90,147,154,53,127,109,126,145, 147,109,83,59,31,99,68,83,
137,90,110,94,59,103,126,88, 132,21,84,127,93,105,137,97, 124,122,120,138,85,68,128,65, 88,44,132,42,58,106,79,34,
124,85,81,119,108,133,128,91, 138,36,114,116,86,113,128,87, 26,102,87,132,98,77,76,98, 95,93,108,117,161,118,149,141,
131,125,92,102,68,135,80,85, 131,41,104,110,136,120,132,145, 74,100,62,135,119,73,95,106, 84,72,119,134,102,130,74,113,
136,71,72,117,124,128,73,106, 138,23,122,64,52,124,130,76, 75,120,120,120,127,85,126,105, 103,120,93,146,119,110,119,124,
59,156,145,150,141,110,137,123, 104,147,127,129,138,125,77,138, 130,135,130,87,148,123,129,136, 129,131,48,15,26,124,110,114,
115,51,73,57,54,120,36,57, 115,11,101,125,70,76,140,93, 150,94,84,126,85,49,83,103, 85,56,128,111,78,103,71,102,
137,112,116,124,112,131,132,116, 126,33,113,86,134,122,127,115, 94,124,117,137,127,116,110,91, 128,86,117,135,114,119,99,120,
116,104,122,83,102,118,81,68, 114,12,145,131,124,108,121,145, 103,127,145,127,115,112,108,119, 140,58,140,108,123,116,98,134,
131,98,133,63,58,136,35,61, 131,29,113,81,78,117,131,74, 128,121,76,127,101,71,79,105, 70,83,126,131,150,127,103,111,
95,145,124,149,145,102,158,144, 73,115,137,128,126,118,39,154, 127,136,124,85,112,136,102,157, 149,155,63,39,21,131,157,85,
121,75,41,91,52,123,59,122, 133,55,87,113,80,75,144,80, 121,95,110,126,123,72,70,79, 101,39,94,99,84,136,68,51,
127,76,124,126,75,95,60,87, 130,36,76,111,110,138,142,109, 125,113,92,127,77,120,87,88, 92,44,116,59,67,146,52,55,
124,94,127,65,59,135,31,83, 154,42,119,47,70,56,99,85, 70,79,69,127,80,57,107,42, 77,38,108,135,113,116,94,95,
134,58,90,56,42,142,76,51, 134,21,125,102,78,135,80,81, 78,60,137,122,61,53,43,84, 147,45,82,70,120,107,56,52,
125,58,101,70,50,142,31,51, 142,44,142,128,93,121,116,115, 115,88,119,127,79,67,100,81, 78,68,109,74,131,115,109,34,
128,20,41,24,44,151,46,58, 143,31,53,37,38,115,104,35, 49,13,50,110,40,29,34,32, 52,77,75,38,140,67,53,62,
30,102,98,113,102,138,130,139, 39,47,119,112,103,104,37,128, 128,103,110,33,80,102,106,125, 86,137,98,43,38,91,95,175,
21,105,132,102,123,141,110,109, 31,168,125,139,150,92,18,94, 86,105,122,38,63,172,77,132, 155,93,48,68,35,60,65,79,
28,139,76,117,98,118,79,120, 117,45,140,50,110,154,88,110, 45,147,116,36,127,99,132,98, 137,124,54,34,56,82,164,143,
35,74,103,109,98,33,86,125, 55,113,149,141,121,121,50,122, 113,105,152,60,154,113,84,61, 128,49,72,16,20,144,58,61,
82,126,110,148,135,61,143,155, 65,78,116,108,135,109,46,91, 122,125,145,67,103,116,124,162, 127,185,84,36,46,110,131,52,
42,137,146,121,136,110,144,167, 67,120,128,125,146,130,34,118, 111,122,138,47,72,152,133,119, 127,160,56,29,36,97,143,120,
};
static const uint8 ced_hires_11[1024] = {
129,46,102,93,87,61,93,55, 96,48,77,76,65,67,91,46, 64,58,113,75,57,62,82,91, 106,70,112,123,68,70,75,108,
113,55,138,144,134,87,155,40, 138,123,135,133,127,142,137,53, 147,126,126,136,132,133,154,124, 123,43,142,128,28,86,28,104,
122,134,145,124,106,123,46,119, 48,128,47,146,45,45,47,127, 41,140,89,43,38,100,46,40, 62,104,61,104,126,110,126,124,
91,125,26,136,96,129,23,115, 19,144,133,118,113,115,63,132, 59,137,94,51,29,117,36,127, 39,136,46,74,118,107,102,110,
89,128,72,40,74,135,21,135, 34,144,57,55,66,55,51,129, 53,127,93,50,32,126,63,29, 37,128,44,72,113,117,128,118,
77,69,127,126,130,77,123,31, 130,136,133,130,123,130,153,82, 138,132,127,134,127,137,125,128, 123,115,114,116,75,59,43,117,
113,116,62,47,39,142,82,137, 41,119,39,88,88,58,47,136, 28,38,72,11,27,108,35,44, 65,155,73,100,130,105,132,111,
93,34,108,139,100,55,67,50, 146,50,129,131,153,145,111,31, 105,138,137,151,137,51,116,127, 146,30,74,76,15,37,10,59,
98,133,33,28,17,143,30,136, 41,115,34,100,131,73,38,123, 12,134,69,24,4,134,39,29, 52,118,53,81,110,140,124,88,
82,128,133,117,145,106,126,85, 134,68,159,123,131,130,125,127, 111,117,130,146,125,44,111,141, 127,118,87,70,96,9,66,50,
92,145,86,40,68,123,37,120, 71,114,90,126,74,74,28,128, 73,120,89,54,119,118,68,72, 51,112,35,63,137,116,114,104,
98,123,104,91,72,135,54,139, 112,130,98,141,114,67,63,136, 86,42,87,61,112,117,101,58, 106,134,41,69,123,123,123,117,
85,137,150,37,54,142,42,126, 36,124,60,30,130,98,12,132, 135,24,89,53,49,98,125,57, 119,105,38,66,134,116,118,107,
98,141,79,89,141,133,98,129, 133,130,72,83,48,92,48,131, 38,31,95,107,139,113,59,69, 40,143,37,65,112,132,107,111,
112,118,69,50,46,149,49,152, 39,130,45,42,40,30,66,118, 54,26,75,14,107,112,49,54, 65,116,73,100,115,143,78,91,
91,69,145,144,140,88,104,98, 127,131,122,136,136,132,115,76, 134,134,127,139,121,152,149,142, 144,83,166,167,31,77,39,99,
82,130,53,59,64,129,54,110, 44,118,48,133,87,91,27,140, 84,148,91,51,122,101,61,86, 50,134,46,84,126,106,112,112,
84,135,121,143,124,127,87,121, 113,142,112,75,123,109,90,133, 90,98,88,84,123,123,125,129, 92,141,44,70,118,108,114,104,
134,108,127,101,121,121,98,125, 121,115,125,94,128,88,98,112, 124,82,126,112,130,134,126,138, 101,122,105,119,92,85,86,126,
77,117,116,48,78,138,27,142, 122,120,126,63,134,48,25,112, 112,39,86,120,154,143,120,143, 28,123,36,63,109,110,108,101,
71,131,58,45,30,128,122,146, 62,130,49,95,90,25,18,143, 48,127,97,99,74,119,33,43, 22,146,30,57,118,112,106,106,
95,104,131,129,128,72,131,81, 132,75,120,142,143,138,133,92, 155,132,138,149,139,63,128,144, 160,88,119,81,75,7,47,66,
102,128,48,47,57,119,70,107, 140,130,80,106,65,90,32,136, 116,123,82,61,117,132,66,43, 55,154,61,89,127,128,105,122,
110,129,58,13,38,138,30,112, 136,126,30,98,89,135,39,126, 55,147,70,32,117,122,68,87, 58,145,57,84,130,114,103,117,
118,115,58,40,45,141,61,156, 51,109,30,30,37,33,59,110, 34,25,81,42,99,147,71,54, 97,128,88,115,120,107,128,103,
102,62,107,133,115,63,64,42, 135,82,88,103,135,156,108,60, 147,137,139,149,144,30,125,110, 68,80,128,101,53,14,38,71,
127,100,145,84,145,70,134,37, 72,39,175,114,120,112,75,120, 76,99,111,119,117,48,91,106, 110,125,102,128,84,56,63,90,
129,77,130,151,138,88,126,88, 117,82,96,136,127,108,124,80, 155,101,115,124,94,99,175,159, 129,107,129,129,106,107,112,114,
98,9,84,76,73,44,76,33, 73,22,61,73,76,66,62,27, 84,63,137,71,74,28,80,84, 106,47,83,88,29,18,15,66,
102,23,80,68,70,48,77,40, 82,24,57,72,77,67,80,46, 70,66,135,70,62,10,75,87, 108,64,76,91,16,23,22,74,
105,19,61,52,70,26,63,47, 80,27,40,55,73,79,73,33, 71,69,127,73,72,12,72,69, 65,55,77,93,18,43,34,72,
104,44,77,65,69,48,79,52, 74,47,58,58,60,52,78,46, 61,55,123,55,55,61,74,74, 89,67,93,106,71,67,69,73,
};
static const uint8 ced_hires_12[1024] = {
79,129,118,114,118,130,135,136, 122,129,124,105,133,117,122,111, 134,119,65,123,122,86,117,114, 121,123,136,124,126,111,73,121,
134,116,126,128,129,122,119,131, 131,128,98,130,123,67,131,120, 126,125,137,42,120,129,124,126, 132,124,121,124,126,120,89,121,
112,133,109,133,111,132,130,71, 99,133,71,58,135,135,120,135, 124,96,123,110,95,72,54,65, 130,119,105,127,127,103,105,127,
118,122,128,106,129,130,78,100, 80,135,92,115,113,114,117,119, 117,117,148,147,127,77,100,118, 129,121,108,127,127,101,91,127,
135,132,136,125,118,124,139,134, 122,120,34,135,116,109,141,104, 122,131,129,5,132,82,135,130, 120,130,119,128,123,119,103,118,
126,133,130,135,119,105,128,126, 122,109,135,125,129,124,128,144, 130,124,127,128,125,138,129,123, 129,123,141,114,116,100,75,111,
126,110,99,100,152,132,105,74, 73,128,77,140,108,97,135,114, 115,76,102,86,74,55,93,128, 128,65,82,127,127,109,97,127,
83,137,114,142,118,131,144,118, 104,132,68,106,127,101,119,118, 124,123,62,94,123,122,117,132, 123,130,118,127,145,96,99,127,
115,129,121,73,117,134,107,125, 107,139,57,129,133,75,111,122, 115,108,124,112,115,65,89,120, 136,89,100,127,127,107,91,162,
116,124,116,129,125,122,126,120, 126,115,136,128,123,153,120,132, 126,124,119,124,121,125,124,127, 124,129,124,115,117,98,87,112,
110,93,120,88,101,100,92,100, 101,103,193,110,109,83,98,97, 81,85,93,129,84,120,83,96, 93,94,86,127,127,127,127,127,
130,129,102,125,118,128,139,116, 89,126,144,112,143,114,130,139, 137,125,99,129,122,105,77,70, 122,122,127,127,144,123,84,127,
140,124,123,126,138,124,114,136, 125,129,136,130,115,106,135,110, 119,120,124,115,130,112,127,128, 111,122,116,121,123,113,80,118,
109,106,107,107,110,111,111,99, 109,127,112,107,106,162,102,138, 105,111,114,99,121,103,86,90, 93,107,100,127,127,170,152,127,
129,122,126,129,130,127,131,138, 125,126,108,134,116,77,116,126, 127,138,131,43,123,110,137,129, 121,138,123,121,123,110,70,118,
108,114,111,100,106,124,104,100, 108,110,114,106,106,96,119,162, 90,141,96,108,93,105,112,107, 100,92,93,127,127,144,144,127,
119,102,134,125,132,132,111,116, 139,135,118,123,93,104,109,107, 109,137,126,109,126,113,120,127, 111,127,122,127,127,96,84,127,
108,121,133,121,111,125,73,107, 148,128,119,114,117,108,124,100, 119,106,116,170,155,38,84,144, 123,58,113,127,127,105,99,154,
57,128,109,135,124,126,134,42, 110,124,72,117,140,133,125,114, 124,133,85,97,99,143,140,113, 129,122,120,127,127,99,87,127,
93,88,77,107,97,101,91,104, 98,96,155,103,106,69,77,79, 81,137,97,206,81,184,134,112, 92,116,64,127,127,127,127,127,
118,85,115,94,120,133,99,119, 127,125,107,92,120,53,74,109, 133,143,137,92,89,114,118,130, 145,133,124,127,127,100,88,127,
93,97,96,121,91,124,100,109, 94,125,112,118,104,82,100,96, 123,97,112,138,108,200,107,142, 135,117,94,127,127,127,127,127,
130,129,119,115,125,130,53,116, 116,133,51,40,120,97,122,101, 108,62,141,108,137,124,103,105, 138,40,104,127,127,109,97,127,
111,131,74,122,129,132,85,105, 144,128,63,62,121,83,101,114, 133,139,115,123,125,105,139,99, 136,139,110,127,127,106,92,127,
132,119,131,119,128,131,116,120, 132,131,135,136,87,90,112,113, 116,132,115,124,125,154,127,132, 94,130,122,127,127,108,106,127,
127,129,109,98,125,122,112,115, 115,128,120,126,142,114,126,103, 132,96,127,105,124,69,102,122, 128,103,120,127,127,98,112,127,
121,120,145,97,119,126,115,132, 89,127,134,127,112,113,133,129, 130,102,128,127,135,64,118,140, 138,124,104,127,127,129,118,127,
127,127,127,127,127,122,127,127, 127,122,127,127,127,127,127,127, 127,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
127,127,127,127,127,118,127,127, 127,118,127,127,127,127,127,127, 145,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
121,124,104,109,109,81,110,97, 104,77,120,119,111,92,119,106, 100,107,107,127,117,127,112,104, 89,124,89,127,127,197,184,127,
115,99,101,103,110,78,92,110, 84,142,93,113,91,65,122,79, 105,120,109,104,120,113,97,90, 86,115,97,127,127,182,217,127,
127,127,127,127,127,122,127,127, 127,124,127,127,127,127,127,127, 127,127,127,127,127,127,127,127, 127,127,127,127,127,127,127,127,
};
static const uint8 ced_hires_13[1024] = {
79,36,84,88,70,26,101,101, 57,93,76,44,82,95,88,77, 79,102,93,112,76,86,62,31, 126,102,50,102,97,110,99,131,
52,43,113,117,112,51,119,123, 107,86,124,118,123,119,119,49, 114,113,112,125,118,105,132,18, 125,60,129,75,77,71,51,90,
99,65,124,123,98,65,96,121, 107,100,76,93,112,140,131,73, 92,127,90,86,132,109,81,54, 57,124,94,100,98,115,106,112,
81,31,42,63,61,107,26,43, 32,76,44,28,57,75,56,165, 61,40,75,79,77,82,47,21, 39,68,76,80,81,88,78,107,
53,55,83,115,118,88,124,129, 135,116,109,113,126,117,121,71, 103,69,120,116,124,140,113,25, 116,121,78,69,55,61,52,95,
75,63,96,27,126,105,92,121, 78,64,55,123,122,101,117,34, 107,33,121,105,112,47,100,18, 60,36,113,61,64,87,64,99,
70,75,100,78,116,77,119,135, 81,90,103,129,128,106,113,57, 91,57,132,124,117,87,117,49, 60,46,37,98,75,96,77,107,
104,148,95,50,41,136,45,51, 72,150,86,83,112,103,30,141, 59,39,43,55,90,129,56,44, 57,84,26,104,80,110,75,126,
72,54,96,117,102,69,79,123, 44,79,60,58,105,135,103,47, 69,152,129,129,112,72,124,24, 118,59,89,72,68,75,69,97,
49,104,110,125,122,126,125,119, 116,94,112,92,115,120,104,110, 129,129,113,124,117,103,117,22, 126,38,90,48,80,82,62,97,
76,70,64,110,64,64,46,62, 48,63,80,67,84,137,129,30, 93,115,59,121,142,110,109,61, 115,61,70,84,90,95,95,95,
100,63,73,65,82,99,97,66, 73,125,80,83,138,86,127,52, 78,109,102,98,105,78,103,62, 112,76,108,111,109,108,113,111,
111,114,86,93,91,76,81,78, 85,82,114,98,92,112,111,94, 98,94,75,90,95,93,90,72, 91,82,103,151,113,118,141,119,
49,138,104,130,121,72,113,92, 114,38,101,119,105,122,105,109, 107,112,87,116,119,68,115,8, 83,24,113,61,68,71,56,94,
114,52,76,101,85,55,85,54, 85,94,75,85,117,114,135,41, 81,88,76,85,140,64,70,51, 76,101,82,105,98,101,102,103,
114,111,126,117,134,117,113,90, 68,92,84,110,117,110,122,91, 91,164,90,122,124,100,108,120, 102,94,88,118,111,108,113,111,
101,137,103,32,71,95,111,96, 97,145,123,85,103,96,79,61, 52,54,86,108,68,157,102,41, 69,71,61,101,99,107,96,119,
73,150,20,28,25,114,26,26, 37,112,46,33,31,22,13,156, 23,49,1,30,13,101,28,20, 32,47,30,71,90,77,68,96,
105,79,95,84,96,64,96,113, 63,97,73,68,104,114,113,65, 121,104,111,119,80,88,85,63, 121,61,65,125,108,129,103,127,
42,53,109,91,120,40,90,107, 68,92,130,92,111,108,134,29, 112,80,111,94,96,54,105,150, 132,48,101,55,61,61,49,94,
106,68,93,88,93,69,69,61, 73,98,117,78,128,133,126,94, 114,78,82,97,141,98,102,65, 75,66,104,102,89,122,99,112,
107,83,114,50,97,149,84,90, 128,149,119,100,113,91,105,79, 132,52,98,96,108,142,94,37, 91,66,96,111,86,94,85,107,
46,66,107,104,113,66,126,117, 117,92,123,126,115,109,116,65, 119,42,129,113,104,53,118,79, 78,127,128,50,61,72,51,99,
121,71,63,62,64,61,69,65, 74,70,74,66,65,54,63,58, 71,84,44,54,50,87,74,66, 113,81,60,152,130,138,134,138,
84,83,129,39,123,95,106,128, 82,84,142,138,113,108,107,57, 118,38,127,106,107,90,120,60, 59,142,45,75,98,85,68,107,
105,90,94,88,71,76,71,67, 72,101,88,64,81,91,95,66, 85,76,67,85,79,82,82,58, 109,96,81,127,113,121,114,123,
73,100,141,107,128,71,80,84, 100,85,121,103,123,126,115,69, 119,72,94,128,109,59,112,26, 83,59,116,115,83,89,73,94,
104,73,90,91,87,88,89,78, 90,76,102,101,111,102,89,43, 82,81,118,96,146,75,100,50, 87,96,96,113,113,106,109,114,
66,58,131,124,106,86,118,114, 132,74,68,119,117,117,117,35, 102,36,125,106,106,24,98,35, 71,128,128,81,81,80,47,96,
73,41,118,138,109,28,78,48, 115,45,110,113,95,133,88,35, 121,38,106,109,103,103,117,18, 44,86,119,71,77,86,76,105,
94,137,71,54,52,142,72,61, 72,154,145,98,92,97,38,93, 58,58,95,69,85,132,127,42, 91,123,55,108,102,106,97,140,
113,65,76,58,62,92,85,62, 80,69,57,64,67,66,51,59, 80,72,50,69,62,68,58,73, 71,60,58,101,115,108,109,118,
};
static const uint8 ced_hires_14[1024] = {
127,73,147,96,119,88,105,109, 120,113,103,118,88,140,101,79, 110,127,68,112,151,94,90,83, 125,83,125,127,127,127,127,127,
88,49,130,116,118,52,139,140, 125,89,134,128,134,134,134,59, 124,140,131,139,131,105,148,19, 145,52,137,100,89,91,77,99,
131,71,141,122,104,66,116,138, 125,103,87,103,122,155,146,83, 102,154,109,100,144,109,97,55, 77,116,102,118,110,122,130,121,
134,25,93,100,106,94,68,101, 54,113,93,18,111,141,103,68, 142,68,136,117,127,146,99,18, 62,18,121,110,88,106,105,109,
87,60,100,113,123,88,143,145, 153,119,119,122,136,131,136,80, 113,95,138,130,136,140,128,26, 136,113,85,93,67,79,77,103,
127,99,154,57,97,71,91,93, 125,69,89,116,117,76,110,81, 149,113,85,77,84,90,75,68, 110,72,120,127,127,120,127,123,
107,127,106,67,34,132,96,32, 46,138,56,96,38,110,100,88, 74,63,26,39,49,130,29,120, 45,34,49,93,90,113,96,118,
140,153,112,49,46,136,65,67, 90,152,96,92,122,118,45,150, 69,64,61,68,102,129,71,44, 77,75,33,128,92,127,99,134,
74,133,89,64,55,143,71,52, 72,150,122,145,125,84,137,124, 106,53,105,97,125,131,90,27, 24,49,71,72,81,88,67,109,
83,109,127,124,128,126,144,135, 134,97,122,101,125,134,119,119, 139,155,131,138,129,102,132,22, 146,29,97,71,91,102,87,105,
88,4,133,137,142,3,28,140, 12,31,52,141,80,31,31,20, 149,46,21,114,133,20,2,90, 69,0,136,73,62,95,79,102,
126,68,90,64,87,99,117,82, 91,127,90,92,147,101,142,60, 88,136,120,111,116,77,118,62, 130,67,116,128,121,114,137,119,
79,15,89,109,132,0,34,27, 136,25,159,134,135,122,132,40, 105,35,121,133,141,23,146,2, 28,0,125,76,68,108,72,91,
84,143,121,128,126,72,133,108, 132,41,111,128,115,136,120,118, 117,138,105,130,131,68,130,9, 103,16,120,86,80,91,80,101,
144,57,93,100,91,55,105,70, 103,96,85,94,127,129,150,50, 91,114,94,98,152,63,85,51, 96,92,89,120,109,108,123,111,
121,154,45,60,63,64,68,39, 59,85,56,120,31,120,40,125, 43,86,21,83,136,120,56,36, 80,36,43,124,106,113,123,115,
121,150,74,14,27,151,56,36, 50,150,69,55,34,102,58,106, 39,82,28,56,34,160,40,31, 75,45,39,131,101,109,119,112,
104,54,77,132,55,34,31,18, 17,42,46,106,0,82,44,80, 36,49,0,155,40,40,6,33, 43,19,25,91,101,99,89,114,
112,141,38,57,37,72,43,25, 101,67,64,150,25,96,40,152, 26,69,15,132,116,149,27,18, 62,32,26,112,97,105,106,108,
78,60,127,91,126,42,111,125, 87,96,141,103,122,124,150,40, 122,106,129,108,108,54,120,151, 152,40,108,78,73,81,74,102,
140,74,110,87,99,70,89,78, 91,102,128,88,139,149,142,104, 123,104,100,110,153,97,117,66, 94,57,111,124,100,129,123,119,
112,104,136,78,126,105,142,117, 127,127,143,142,137,115,105,119, 116,69,120,135,123,76,130,29, 62,18,144,114,88,105,106,108,
82,72,125,103,119,67,146,134, 136,96,134,136,126,124,132,75, 129,68,147,127,116,53,133,80, 97,119,135,74,72,91,76,106,
131,76,81,61,69,61,88,82, 92,73,84,76,75,69,78,66, 81,110,61,67,61,85,87,66, 132,71,67,154,139,145,139,146,
84,133,98,52,42,158,37,23, 89,160,53,107,63,86,87,104, 70,38,30,103,87,98,78,0, 32,26,51,80,58,80,75,91,
95,14,130,98,109,14,63,29, 116,13,156,109,122,149,89,39, 70,49,106,128,105,19,149,25, 43,4,137,91,82,95,86,110,
109,107,159,107,134,73,101,102, 119,89,132,114,134,142,131,80, 129,98,112,142,121,59,127,27, 103,51,123,138,94,108,99,102,
123,108,129,80,86,116,133,102, 135,85,156,146,124,112,121,59, 80,90,122,135,124,64,156,40, 93,40,143,127,110,116,125,119,
102,64,148,123,112,87,139,131, 152,77,79,129,128,132,133,45, 111,62,143,119,118,22,113,35, 90,119,135,104,92,99,72,104,
109,48,136,138,115,29,99,66, 134,49,121,124,106,149,104,46, 131,63,124,123,115,103,132,19, 63,78,126,96,88,105,101,112,
117,136,38,70,20,133,49,30, 43,174,47,36,24,27,9,70, 34,75,10,15,28,140,32,25, 69,25,32,117,95,106,112,109,
136,54,77,34,51,77,76,61, 86,49,60,57,54,66,64,51, 69,93,44,77,50,54,57,55, 81,42,72,112,121,115,118,126,
};
static const uint8 ced_hires_15[1024] = {
128,67,142,105,122,81,101,108, 115,105,99,113,90,139,98,75, 108,127,69,108,148,90,86,94, 122,79,127,128,128,128,128,128,
69,43,125,125,121,45,135,139, 120,81,130,123,135,133,130,54, 122,137,129,135,128,100,144,30, 140,48,139,91,84,84,66,98,
115,65,136,131,106,59,112,137, 120,95,82,98,123,154,142,78, 100,151,107,96,141,104,93,66, 72,112,103,112,105,119,121,120,
117,20,88,109,109,87,64,100, 48,105,89,13,112,140,99,63, 140,65,134,113,124,141,95,29, 57,14,123,101,83,100,94,108,
68,54,95,122,126,81,139,144, 148,111,115,117,137,130,132,75, 111,92,136,126,133,135,124,37, 131,109,87,85,62,73,66,102,
128,93,149,66,99,65,86,92, 121,64,84,111,119,76,106,76, 146,112,85,74,81,85,71,79, 108,70,121,126,126,117,128,122,
88,121,101,75,36,126,91,31, 42,131,51,91,40,109,96,83, 71,59,25,36,46,125,25,131, 41,30,50,84,85,105,85,117,
121,147,107,57,49,130,60,66, 85,145,91,87,124,117,41,145, 66,61,60,65,99,124,67,55, 72,71,35,119,87,122,89,133,
57,127,84,73,58,136,67,51, 67,142,118,140,126,83,133,119, 104,49,103,93,122,126,86,38, 19,45,73,63,77,81,56,108,
65,103,122,132,130,120,140,134, 129,89,118,96,126,133,115,114, 137,152,130,134,126,97,128,34, 141,25,99,63,86,95,76,104,
69,0,128,145,144,0,23,138, 6,23,47,135,81,30,27,14, 146,42,19,110,129,14,0,100, 64,0,137,63,57,87,67,101,
115,62,85,72,89,93,112,81, 86,120,86,87,149,100,138,56, 85,132,119,108,113,72,114,73, 126,63,117,122,116,111,127,118,
60,9,84,117,134,0,29,27, 131,18,154,129,137,121,128,35, 102,31,120,130,138,19,142,14, 24,0,126,67,63,101,61,91,
65,137,116,137,129,65,129,107, 127,33,107,123,116,135,116,113, 115,135,103,126,128,63,126,20, 98,12,122,77,75,84,70,100,
130,51,88,108,93,49,100,69, 98,89,80,89,129,128,146,44, 88,110,92,95,149,58,81,63, 91,88,91,114,104,105,116,110,
111,148,40,69,66,57,65,38, 55,77,52,115,34,119,36,120, 41,83,19,79,133,115,53,47, 75,32,45,118,101,111,112,114,
109,144,70,22,29,145,51,35, 45,143,64,50,37,101,54,101, 36,78,28,54,31,155,37,42, 71,41,40,124,96,106,108,111,
84,47,72,141,57,27,25,15, 12,34,40,100,1,81,40,74, 33,45,0,151,36,34,1,42, 38,14,26,81,95,91,77,113,
96,135,34,65,40,66,38,23, 96,60,59,145,27,95,36,147, 23,65,15,129,113,144,24,29, 58,28,27,104,92,101,95,107,
59,54,122,100,129,35,107,124, 82,88,137,98,123,123,146,35, 120,103,127,104,105,49,116,162, 147,36,110,70,68,74,63,101,
123,67,105,95,101,63,84,76, 86,94,123,82,140,147,138,98, 120,100,98,106,149,91,112,76, 89,52,112,117,95,126,112,118,
96,98,131,86,128,99,137,116, 122,120,138,137,139,114,101,114, 113,65,119,132,120,71,126,41, 58,14,145,105,83,101,95,107,
63,66,120,112,122,60,142,133, 131,88,130,131,127,123,128,70, 127,65,145,123,113,48,129,91, 92,115,137,65,68,84,65,105,
132,70,76,69,72,54,84,81, 87,66,80,71,77,68,74,62, 78,106,61,63,58,81,84,77, 127,67,68,155,136,142,140,145,
65,127,93,61,45,151,33,22, 84,152,49,102,64,85,83,99, 68,35,28,99,84,93,74,2, 27,22,53,71,53,74,64,90,
76,8,125,107,112,8,59,28, 111,6,152,104,123,148,85,34, 68,46,104,124,102,13,145,36, 38,2,139,82,77,88,75,110,
90,101,154,116,137,66,97,101, 114,81,128,109,135,141,127,75, 127,95,110,138,118,54,123,38, 98,47,125,130,89,101,88,102,
117,102,124,88,89,109,129,101, 130,77,152,141,125,111,117,55, 79,87,120,131,121,58,152,51, 90,36,145,122,105,113,116,118,
83,58,143,132,115,81,134,130, 147,70,74,124,129,131,129,40, 109,59,141,116,115,18,109,46, 85,115,137,95,87,92,61,103,
90,42,131,147,118,23,95,65, 129,41,117,119,107,148,100,41, 129,60,122,119,112,98,128,30, 59,74,128,87,83,99,90,112,
102,130,34,79,23,126,45,29, 38,166,43,32,26,27,5,65, 30,72,8,11,24,135,28,36, 64,21,34,108,90,103,101,108,
121,48,73,42,53,70,71,61, 81,41,56,52,55,65,60,46, 67,89,42,73,48,49,53,67, 76,38,74,106,116,112,115,125,
};
static const uint8 ced_hires_16[1024] = {
126,77,94,99,85,77,105,111, 93,98,106,78,81,99,92,92, 100,116,99,119,82,86,89,94, 132,90,87,127,125,126,126,131,
97,47,126,130,125,63,130,137, 117,95,136,122,124,126,123,57, 128,120,120,134,123,99,146,65, 132,56,132,97,96,97,97,110,
143,93,132,131,106,93,114,132, 110,112,123,95,107,141,130,103, 112,132,92,86,131,103,106,111, 132,107,103,143,142,143,143,130,
128,105,112,105,109,106,116,109, 113,112,120,113,104,107,99,110, 114,124,100,102,105,111,112,115, 125,111,116,128,128,128,128,128,
97,58,96,127,131,95,134,142, 146,126,121,116,127,123,126,86, 117,87,128,126,128,135,126,65, 123,114,81,98,96,97,97,110,
157,107,120,109,115,107,128,115, 124,121,137,109,95,107,86,117, 126,146,89,100,96,117,120,125, 146,116,117,157,156,157,157,139,
157,107,120,109,115,107,128,115, 124,121,137,109,95,107,86,117, 126,146,89,100,96,117,120,125, 146,116,117,157,156,157,157,139,
120,155,109,72,78,142,91,78, 87,159,100,85,112,108,49,156, 89,109,52,63,93,123,83,88, 109,79,80,120,119,120,120,118,
119,70,107,128,115,75,90,135, 86,88,99,71,105,140,107,80, 93,159,136,138,115,79,136,87, 118,78,88,120,118,119,119,118,
92,112,123,137,135,133,135,132, 127,104,124,95,116,126,108,126, 143,137,121,134,121,97,130,60, 133,51,92,93,91,92,92,106,
130,80,93,121,88,80,101,88, 97,94,110,82,83,142,131,90, 104,119,62,128,144,100,121,98, 119,89,90,130,129,130,130,123,
148,98,111,100,106,98,119,106, 115,127,128,100,131,98,124,108, 117,137,102,96,97,108,111,116, 137,107,108,148,147,148,148,135,
149,100,112,101,108,100,120,107, 116,113,129,101,88,99,99,110, 118,139,82,93,88,109,112,117, 138,108,110,150,148,149,149,139,
98,145,117,142,133,78,123,105, 125,62,112,122,105,128,109,124, 120,119,94,125,123,58,128,66, 97,57,115,98,97,98,98,112,
145,96,108,110,104,96,116,103, 112,109,125,97,110,112,132,106, 114,135,78,89,137,105,108,113, 134,104,106,146,144,145,145,132,
153,111,124,118,134,108,124,111, 120,117,133,105,101,103,113,113, 122,159,85,115,115,113,116,121, 142,112,113,153,152,153,153,136,
128,112,119,112,116,113,123,116, 120,119,127,120,111,114,106,117, 120,128,106,108,111,117,118,121, 128,117,122,128,128,128,128,128,
120,158,83,72,78,121,91,78, 87,121,100,72,58,70,49,172, 88,108,51,62,58,95,82,87, 108,78,79,119,118,119,119,122,
146,96,109,98,104,96,117,120, 113,110,126,98,94,116,108,106, 130,134,109,118,84,105,108,113, 134,104,105,145,144,145,145,137,
98,58,122,104,132,48,101,121, 75,101,142,96,111,115,138,58, 126,86,118,103,100,57,118,161, 139,56,102,97,96,97,97,111,
139,89,102,96,102,89,110,97, 106,103,124,91,124,135,126,104, 122,127,80,99,141,98,106,106, 127,97,98,138,137,138,138,129,
155,105,128,107,113,105,126,113, 122,119,135,107,93,105,84,115, 133,143,86,97,93,114,117,122, 143,113,114,154,153,154,154,139,
99,72,121,117,126,72,137,131, 129,103,136,130,116,116,121,79, 134,87,137,123,109,58,132,89, 87,121,131,98,97,98,98,111,
154,104,117,106,112,104,125,112, 121,118,134,106,92,104,83,114, 122,142,85,96,92,113,116,121, 142,112,113,153,152,153,153,139,
156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
150,101,113,102,109,101,121,108, 117,114,130,102,89,100,80,111, 118,139,82,93,88,109,112,117, 138,108,110,150,148,149,149,140,
118,107,154,120,140,68,89,96, 111,92,132,106,123,133,119,78, 133,106,101,137,113,77,125,85, 106,76,118,122,116,117,117,121,
149,99,112,101,107,99,120,107, 116,113,129,101,102,99,83,109, 117,137,115,96,139,108,111,116, 137,107,108,148,147,148,148,135,
102,62,144,137,119,94,129,128, 144,82,82,123,118,124,122,62, 116,90,133,116,111,61,112,69, 90,121,131,101,100,101,101,110,
156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
156,106,119,108,114,106,127,114, 123,120,136,108,94,106,85,116, 124,144,87,98,94,115,118,123, 144,114,115,155,154,155,155,140,
137,88,100,89,96,96,108,95, 104,101,117,89,76,87,67,98, 105,126,69,80,75,96,99,104, 125,95,97,137,135,136,136,123,
};
static const uint8 ced_hires_17[1024] = {
89,91,92,133,130,37,125,74, 42,95,144,140,137,137,121,42, 123,75,132,124,133,87,128,83, 74,53,119,94,87,91,93,108,
102,61,121,126,115,65,141,137, 130,96,110,126,137,135,142,73, 128,143,129,129,127,116,147,95, 154,82,141,107,100,103,106,114,
152,89,132,131,102,84,114,134, 128,107,81,104,125,155,153,95, 103,154,108,87,139,120,95,146, 137,146,100,157,150,154,156,136,
130,66,62,72,62,122,86,62, 83,84,58,60,65,91,78,190, 73,115,92,83,85,91,68,123, 115,94,83,135,128,131,134,127,
103,75,93,124,121,103,146,143, 159,127,96,121,140,133,145,96, 118,98,137,121,133,153,128,96, 146,146,90,108,101,104,107,114,
111,83,105,33,129,120,114,135, 102,75,39,131,136,117,141,54, 122,96,138,110,121,62,115,104, 96,75,125,116,109,112,115,120,
131,94,108,84,119,91,141,148, 104,99,90,137,142,121,136,74, 104,117,149,128,125,99,131,125, 116,95,74,136,129,133,135,128,
109,71,69,85,106,85,106,131, 62,57,93,110,101,126,73,52, 116,94,127,130,102,60,143,102, 94,73,52,114,107,118,113,119,
100,147,79,74,50,157,72,48, 76,157,98,143,128,85,145,139, 110,85,103,87,121,143,89,93, 85,84,74,105,98,101,104,112,
97,124,119,134,125,141,147,132, 139,104,99,100,129,135,127,134, 143,159,130,128,125,115,131,91, 155,61,101,102,95,99,101,109,
101,37,124,147,139,33,57,137, 54,39,29,139,83,34,39,44, 153,86,16,104,129,52,39,151, 86,65,140,106,99,102,105,116,
159,95,91,81,81,113,115,91, 112,133,87,89,150,97,149,102, 102,144,118,100,111,110,113,152, 144,123,117,164,157,160,163,142,
86,23,75,47,132,18,43,105, 39,54,141,105,128,124,110,29, 89,72,105,147,131,38,115,80, 71,50,78,91,84,88,90,107,
103,158,113,138,124,86,135,105, 137,41,88,127,119,137,128,133, 120,142,104,120,127,81,129,97, 111,67,124,108,101,105,107,116,
156,93,94,108,84,88,113,88, 109,99,85,92,130,128,156,99, 99,142,88,85,147,108,94,150, 141,120,99,161,154,158,160,138,
98,50,162,115,131,30,54,149, 51,46,135,121,106,128,119,41, 135,83,109,116,138,49,147,91, 83,62,136,103,96,99,102,111,
160,171,92,82,82,172,116,92, 113,164,88,90,85,109,82,129, 97,139,69,66,71,175,92,147, 139,118,97,159,152,155,158,137,
129,176,77,84,94,159,85,102, 82,160,57,105,70,85,71,176, 89,108,48,73,73,173,81,116, 108,87,76,128,121,124,127,122,
180,139,119,109,109,119,143,119, 140,135,125,117,112,120,109,130, 124,166,112,93,98,160,119,174, 166,145,124,180,179,177,180,152,
156,139,88,78,116,166,138,88, 109,177,94,86,81,89,111,115, 93,135,95,88,77,162,98,171, 135,114,93,155,148,151,154,133,
156,92,106,103,101,88,112,88, 114,119,110,91,149,156,157,125, 129,135,101,102,152,114,117,143, 135,114,116,155,148,151,154,133,
152,109,131,74,109,171,109,112, 159,167,115,117,135,115,136,115, 148,132,118,103,119,157,112,140, 131,110,109,151,144,148,150,134,
112,94,124,121,124,90,156,139, 149,111,118,142,137,133,148,98, 137,91,149,121,116,67,136,145, 107,155,143,111,104,107,110,113,
180,119,115,104,105,114,139,114, 135,120,111,113,108,115,104,125, 119,162,92,88,93,128,114,170, 161,140,119,178,174,174,178,149,
128,111,146,50,134,118,137,150, 112,103,137,155,135,132,138,87, 135,108,147,113,118,105,137,126, 107,169,65,127,120,124,126,122,
109,46,52,31,32,41,66,41, 62,47,127,60,65,52,41,52, 46,89,19,130,30,55,41,97, 88,67,66,108,101,105,107,132,
131,129,158,124,139,93,111,106, 131,102,116,120,145,150,146,102, 136,111,114,135,120,77,129,119, 110,89,130,146,123,127,129,123,
171,107,103,101,93,108,127,103, 124,109,99,114,130,122,116,114, 108,150,135,102,155,116,111,158, 150,129,108,170,163,166,169,141,
115,87,147,141,117,109,149,136, 164,92,64,135,139,141,148,71, 119,94,145,113,118,60,116,102, 99,155,143,114,107,110,113,112,
180,125,121,120,111,120,145,120, 141,126,117,119,130,121,126,131, 125,168,98,94,99,134,120,176, 167,146,125,180,179,177,180,152,
108,45,99,103,139,40,96,117, 61,87,85,145,139,127,138,51, 127,88,152,133,136,64,131,96, 87,66,135,107,100,104,106,112,
157,94,95,80,80,109,114,90, 119,96,86,88,83,91,80,101, 95,137,67,64,69,103,90,145, 137,116,95,154,150,150,154,125,
};
static const uint8 ced_hires_18[1024] = {
100,47,105,106,88,37,119,122, 76,90,99,47,87,103,100,98, 95,121,106,126,89,84,80,50, 146,101,49,119,100,126,121,130,
74,55,134,135,131,65,138,145, 126,83,147,122,129,128,131,71, 130,133,126,138,131,103,151,36, 145,59,129,97,84,83,73,90,
123,78,145,142,117,80,115,143, 127,97,99,98,117,149,143,95, 108,146,104,100,145,107,101,72, 83,123,94,119,107,123,126,111,
102,42,63,81,78,121,44,64, 50,73,66,32,62,84,68,187, 77,62,89,92,89,80,66,40, 63,66,76,102,89,103,100,106,
72,67,105,133,136,102,142,150, 155,114,132,117,131,125,133,93, 119,88,133,130,137,139,132,46, 137,120,78,89,64,76,73,94,
94,75,117,44,144,118,110,142, 98,61,78,127,127,109,129,55, 123,48,134,119,125,45,118,38, 80,35,112,83,72,101,83,96,
95,87,122,95,134,91,137,156, 101,88,126,133,133,114,125,79, 107,76,145,138,130,85,136,69, 79,46,38,117,85,112,100,104,
124,160,116,69,60,150,64,73, 92,147,109,87,118,112,42,163, 75,62,57,68,103,128,75,63, 80,83,28,120,89,124,96,125,
94,66,118,135,120,83,97,144, 63,76,82,62,110,143,115,69, 85,171,142,143,125,71,143,43, 138,58,89,97,78,95,93,94,
67,116,131,143,140,140,143,140, 136,91,135,96,120,128,115,131, 145,148,127,138,130,101,135,42, 146,37,89,70,86,94,79,96,
101,82,86,128,82,78,65,84, 68,60,103,72,89,145,141,49, 109,135,73,134,155,108,128,81, 135,61,70,108,97,107,117,96,
123,75,94,83,100,113,116,87, 92,122,102,87,143,94,139,71, 93,129,116,111,117,75,121,80, 131,72,108,126,118,118,133,111,
125,126,107,111,110,90,98,100, 104,78,136,102,98,121,122,116, 113,113,88,103,108,91,109,90, 113,82,103,166,118,129,154,121,
71,150,125,148,139,85,131,113, 134,35,124,123,110,130,117,130, 123,131,100,130,132,66,133,26, 103,24,112,82,78,81,75,92,
134,66,98,119,103,70,104,77, 104,91,98,89,123,122,146,64, 97,108,89,98,153,65,88,72, 98,100,82,121,109,113,126,105,
134,123,148,135,153,131,131,112, 86,89,109,114,122,118,134,113, 106,183,104,135,137,97,127,140, 120,93,88,129,122,120,132,113,
95,138,100,94,128,146,84,59, 64,158,26,72,140,108,64,96, 48,37,129,66,66,162,79,20, 36,60,93,75,70,90,70,98,
95,163,41,47,45,129,41,47, 56,110,67,38,37,31,25,179, 45,69,19,50,32,105,52,43, 62,51,33,101,100,101,98,98,
122,92,117,103,114,78,115,136, 82,95,96,73,110,123,125,87, 143,128,130,139,99,92,109,87, 147,65,68,148,119,143,129,127,
68,66,131,110,138,55,109,130, 88,90,153,97,117,117,146,52, 134,105,130,114,115,58,129,176, 158,53,106,82,71,82,78,96,
129,80,114,106,112,83,88,82, 92,96,140,82,134,142,139,116, 135,103,101,116,160,101,127,91, 102,70,110,124,105,134,125,113,
123,95,135,69,116,163,102,112, 149,147,143,105,119,100,118,101, 153,80,118,115,127,146,119,62, 116,70,102,133,102,113,117,109,
70,78,129,122,132,80,145,139, 137,90,147,130,121,118,129,87, 141,67,148,133,123,58,143,105, 104,132,134,80,74,95,79,100,
136,82,86,79,83,76,90,87, 92,67,102,69,70,65,74,80, 92,115,66,71,68,91,96,94, 140,85,71,156,136,150,142,140,
104,96,151,58,141,110,125,150, 102,82,165,143,118,117,119,80, 140,61,146,125,125,94,144,85, 85,147,50,100,110,107,95,109,
122,103,115,106,89,90,88,87, 92,99,110,70,86,100,108,86, 107,107,86,104,98,85,106,87, 134,101,84,138,122,134,133,128,
93,113,163,126,146,86,99,107, 120,83,144,108,129,135,127,92, 141,96,113,148,128,63,136,50, 109,64,121,140,95,106,100,97,
128,86,112,109,106,102,108,101, 111,75,126,105,117,111,101,66, 104,108,137,116,165,81,125,82, 114,102,102,129,126,121,134,116,
89,71,152,143,125,101,137,136, 153,72,92,124,123,126,129,57, 123,62,145,126,125,29,123,61, 97,133,134,107,94,103,79,98,
99,71,108,117,119,66,122,99, 108,25,83,132,126,131,130,88, 120,83,124,123,119,71,105,40, 88,140,139,89,119,109,90,120,
109,146,119,71,82,143,99,110, 107,147,49,135,131,127,53,105, 80,80,69,92,140,135,125,42, 60,114,37,69,92,91,80,109,
129,78,95,77,81,106,104,83, 99,66,82,66,73,73,63,80, 101,97,69,88,80,74,80,98, 98,65,64,114,130,119,121,120,
};
static const uint8 ced_hires_19[1024] = {
98,93,104,135,134,69,120,71, 84,99,150,144,141,141,121,77, 123,96,135,127,139,85,130,83, 94,88,120,97,97,97,97,113,
110,85,134,128,119,81,138,142, 127,104,115,130,141,138,142,90, 128,119,133,131,132,117,148,95, 129,101,142,109,109,109,109,119,
145,120,126,112,107,116,133,123, 131,123,115,115,110,142,136,124, 123,143,99,95,128,132,120,130, 141,135,125,144,144,144,144,134,
134,109,115,93,96,120,122,107, 120,112,104,104,99,106,91,186, 112,132,88,84,92,121,109,119, 130,124,114,133,133,133,133,130,
111,86,102,124,125,107,142,148, 156,131,104,126,144,136,144,90, 119,109,140,123,138,153,129,96, 122,136,91,110,110,110,110,118,
118,93,119,78,133,123,107,139, 104,97,88,134,139,118,140,98, 122,117,141,111,125,105,113,103, 114,109,123,117,117,117,117,124,
135,110,116,95,117,106,124,148, 121,114,105,136,141,117,132,115, 114,134,148,125,126,122,125,120, 131,126,115,134,134,134,134,130,
117,92,98,86,109,98,105,136, 103,95,97,112,105,129,74,96, 110,115,130,131,106,104,144,102, 113,107,97,116,116,116,116,123,
108,150,89,67,70,160,96,81, 94,161,103,147,132,80,144,139, 109,106,105,88,126,143,83,93, 104,98,88,107,107,107,107,117,
106,127,132,135,129,144,143,138, 136,107,104,103,133,139,126,134, 143,136,133,131,131,114,133,91, 130,96,102,105,105,105,105,114,
109,84,137,148,143,80,97,142, 95,87,79,143,84,81,66,88, 152,107,63,106,134,96,84,104, 105,99,141,108,108,108,108,120,
146,121,127,106,108,117,135,119, 132,125,116,116,133,118,126,126, 125,145,101,96,104,133,121,131, 142,137,126,145,145,145,145,136,
95,70,86,54,136,66,83,111, 81,73,147,109,132,128,110,74, 88,93,108,151,137,82,117,80, 91,85,75,94,94,94,94,111,
112,161,124,140,127,83,131,110, 135,90,92,132,123,141,128,133, 120,110,106,123,133,99,130,97, 108,102,125,111,111,111,111,121,
145,120,126,105,107,116,134,118, 131,124,115,115,115,117,137,125, 124,144,100,95,132,132,120,130, 141,136,125,144,144,144,144,135,
148,123,129,107,115,119,136,121, 134,126,118,118,113,120,115,127, 126,146,102,103,111,135,123,133, 144,138,128,147,147,147,147,136,
147,153,128,106,109,118,135,120, 133,153,117,117,112,119,104,126, 126,146,102,108,106,165,123,133, 144,138,128,147,147,147,147,136,
132,174,113,92,94,156,121,105, 118,160,102,102,97,104,90,172, 112,132,88,83,91,175,108,118, 129,124,113,132,132,132,132,127,
152,127,133,112,114,123,141,125, 138,131,122,122,117,124,110,132, 132,152,108,103,111,140,128,138, 149,144,133,152,152,152,152,138,
117,92,136,107,132,88,105,132, 103,110,127,110,134,133,162,96, 133,116,138,108,116,105,127,176, 144,108,118,117,117,117,117,120,
148,123,129,107,110,119,136,121, 134,126,118,118,135,141,139,127, 127,147,103,99,147,136,124,134, 145,139,129,148,148,148,148,134,
148,123,129,107,110,162,136,121, 144,159,118,118,123,120,120,127, 142,147,113,99,117,151,124,134, 145,139,129,148,148,148,148,135,
117,92,133,120,125,98,149,141, 144,116,122,143,138,132,145,97, 142,117,157,127,125,105,141,103, 114,150,148,117,117,117,117,120,
153,128,134,112,115,124,141,126, 139,131,123,123,118,125,110,132, 132,152,108,104,112,141,129,139, 150,144,134,153,153,153,153,138,
132,107,153,92,134,113,121,151, 118,111,137,154,135,129,133,112, 140,132,153,117,125,120,141,118, 129,163,113,132,132,132,132,127,
113,88,94,73,75,84,102,86, 99,92,127,83,78,85,71,93, 93,113,69,135,72,101,89,99, 110,105,94,113,113,113,113,129,
135,125,166,120,137,106,124,108, 121,114,115,115,143,149,142,115, 138,135,119,140,128,123,131,121, 132,127,131,135,135,135,135,129,
151,126,132,111,113,122,140,124, 137,130,121,121,116,123,109,131, 131,151,114,102,138,139,127,137, 148,143,132,151,151,151,151,137,
120,95,157,140,118,108,142,138, 158,99,90,136,140,141,145,100, 121,120,153,120,127,108,119,106, 117,149,148,120,120,120,120,119,
136,111,140,151,118,107,124,109, 137,114,106,126,111,155,113,115, 138,135,129,118,120,124,137,122, 133,127,132,136,136,136,136,131,
151,146,132,110,113,145,139,124, 137,154,121,121,116,123,108,130, 130,150,106,102,110,139,127,137, 148,142,132,151,151,151,151,137,
146,121,127,105,108,117,134,119, 132,124,116,116,111,118,103,125, 125,145,101,97,105,134,122,132, 143,137,127,146,146,146,146,131,
};
static const uint8 ced_hires_20[1024] = {
111,51,50,66,67,37,61,60, 62,52,151,76,117,107,89,44, 64,85,123,112,43,48,109,55, 80,47,73,105,103,120,107,126,
99,78,66,40,113,81,103,136, 51,33,85,110,79,133,68,53, 121,80,125,133,107,49,144,49, 79,46,58,104,94,114,94,119,
82,89,103,144,141,45,125,85, 60,80,142,139,139,140,118,46, 133,65,134,127,134,77,126,29, 65,28,130,79,73,84,74,99,
148,112,111,112,105,115,135,114, 133,109,103,112,104,109,99,132, 123,145,97,101,94,117,110,138, 144,123,128,148,147,147,148,137,
97,70,96,138,128,103,151,149, 163,109,95,119,143,133,146,96, 122,107,138,122,134,143,125,67, 141,129,98,104,95,92,94,114,
112,73,108,62,136,120,118,141, 105,57,53,129,139,116,142,70, 126,92,139,111,122,64,111,67, 100,63,134,104,102,110,102,119,
126,53,74,82,77,64,78,76, 74,51,58,72,68,60,68,61, 85,102,50,148,65,58,59,77, 97,64,90,122,116,120,117,129,
94,50,95,141,134,56,69,127, 60,37,151,133,141,146,122,58, 115,74,138,121,145,52,105,44, 78,35,128,96,85,89,89,105,
93,143,83,89,63,158,83,60, 83,139,98,142,132,86,147,141, 115,84,105,89,123,134,85,67, 69,69,82,94,100,99,89,116,
92,119,122,149,133,142,152,139, 144,87,98,98,132,136,128,136, 148,160,131,130,127,105,129,63, 151,58,111,91,104,103,95,113,
148,106,116,121,106,102,130,108, 127,89,97,117,103,117,112,113, 133,145,97,94,107,111,109,132, 144,121,140,148,147,147,148,137,
96,26,70,54,136,23,56,118, 57,39,143,109,136,129,114,24, 102,75,104,149,132,26,115,40, 62,32,99,95,91,85,89,104,
122,116,63,61,63,156,88,58, 77,162,52,66,68,77,80,112, 84,106,43,53,46,123,55,75, 100,67,77,125,117,120,120,120,
137,128,86,79,104,158,111,95, 93,154,68,90,84,84,82,113, 113,121,63,81,93,155,94,91, 115,82,99,136,132,131,136,128,
91,40,166,130,139,40,74,156, 53,33,137,119,108,132,120,47, 139,70,109,117,140,49,148,47, 74,31,142,89,83,85,84,103,
113,165,84,84,91,133,107,85, 101,142,80,94,94,82,76,158, 92,97,86,89,67,171,112,67, 91,63,89,116,108,115,112,117,
84,151,102,50,102,126,72,103, 59,149,74,148,115,116,104,148, 125,78,106,107,133,146,137,43, 60,158,87,85,79,92,80,104,
115,72,78,158,69,65,71,53, 67,50,47,103,38,82,62,99, 72,98,29,152,56,69,47,84, 93,67,72,118,125,119,113,127,
116,167,96,89,95,156,80,88, 85,135,64,95,66,79,70,161, 97,98,65,84,85,166,84,67, 92,59,94,117,117,113,112,116,
95,66,122,116,132,60,119,129, 93,85,117,101,129,126,160,66, 136,118,134,105,111,69,122,180, 161,71,127,101,101,101,96,115,
148,107,121,119,120,108,132,120, 135,112,126,122,123,123,121,115, 140,147,122,115,120,131,127,129, 146,120,138,149,149,149,149,138,
145,100,125,89,108,164,120,114, 156,141,106,107,130,110,130,109, 154,126,119,105,123,147,113,96, 136,98,123,146,132,131,136,131,
96,81,121,129,125,83,154,139, 147,86,111,134,134,126,142,92, 143,98,152,124,120,68,135,108, 111,140,154,101,101,106,99,117,
148,101,97,103,91,93,119,102, 119,82,86,90,96,95,97,105, 112,145,87,80,83,109,104,119, 153,110,113,160,153,158,154,146,
96,28,42,33,46,35,53,45, 49,31,126,40,50,53,52,43, 68,80,39,131,35,47,41,50, 74,51,47,99,104,105,97,134,
148,137,108,111,112,125,128,118, 139,87,95,116,95,112,110,149, 117,147,86,88,103,140,105,130, 146,146,127,149,149,149,149,138,
148,129,121,142,101,121,132,114, 128,114,99,114,126,128,120,128, 126,147,104,99,97,132,116,156, 146,121,126,149,149,149,149,138,
94,39,97,117,138,45,88,117, 67,38,75,139,136,117,133,55, 139,78,153,138,143,54,128,55, 79,53,151,97,89,100,92,107,
108,73,144,149,117,103,147,136, 162,65,59,127,136,135,143,70, 126,97,148,117,122,55,115,79, 109,140,154,117,114,112,100,115,
148,128,120,108,115,137,133,110, 129,119,99,112,110,115,116,137, 127,147,98,107,93,131,122,130, 146,149,127,149,149,149,149,138,
90,141,92,62,123,145,98,94, 71,157,54,115,84,131,112,151, 121,73,99,126,103,146,129,51, 68,144,73,93,87,100,88,107,
137,80,87,78,75,97,105,85, 110,75,63,75,82,82,81,90, 99,126,66,86,72,87,81,105, 118,85,106,131,142,134,137,133,
};
static const uint8 ced_hires_21[1024] = {
83,35,83,86,68,31,99,99, 55,99,74,45,80,93,87,75, 79,100,91,111,76,86,61,33, 125,101,49,100,98,111,105,129,
59,43,112,116,111,56,117,121, 105,92,122,118,122,117,117,48, 113,112,111,124,119,105,131,14, 124,58,128,80,75,64,53,89,
102,65,123,122,96,71,94,119, 105,106,73,93,110,138,129,71, 91,126,89,85,132,109,80,55, 63,122,92,108,98,117,104,108,
87,29,40,62,58,112,27,40, 29,82,41,27,55,73,54,164, 60,39,74,78,77,82,45,14, 40,65,75,85,83,89,81,104,
56,54,82,113,116,92,121,126, 133,122,107,112,124,114,119,69, 102,67,118,115,124,140,111,23, 115,119,76,70,57,61,58,92,
78,62,95,25,124,109,90,119, 76,70,53,123,120,99,116,31, 107,29,119,104,112,46,98,15, 58,34,111,65,65,82,61,97,
81,74,99,75,114,81,117,133, 79,96,101,129,126,104,112,55, 91,50,130,123,117,86,115,48, 57,41,32,98,76,95,82,105,
107,147,94,49,39,141,43,49, 70,156,84,83,111,101,28,140, 58,35,42,54,90,129,55,41, 53,82,24,102,79,105,77,124,
75,53,95,115,100,73,77,121, 43,85,59,58,103,133,102,46, 69,150,127,128,112,72,122,25, 117,58,87,80,75,79,76,97,
52,103,109,123,121,131,123,117, 114,100,110,92,113,118,102,108, 129,127,111,123,117,103,115,21, 125,36,88,52,79,81,65,95,
87,70,63,109,62,69,46,61, 46,69,77,67,83,135,128,27, 93,114,58,120,143,110,108,59, 114,61,69,92,90,96,100,95,
109,61,71,65,81,104,96,64, 70,131,79,83,136,84,125,49, 77,107,101,97,105,80,101,61, 111,75,106,115,115,112,116,109,
112,114,84,92,90,82,79,75, 83,89,111,97,91,110,108,93, 97,93,73,90,96,92,89,68, 88,77,101,154,114,122,143,121,
53,137,103,128,119,76,111,90, 112,44,99,119,103,120,104,107, 107,110,85,115,119,68,113,4, 82,21,111,63,72,62,54,91,
115,54,75,100,84,61,84,54, 83,100,70,85,116,112,133,43, 80,88,74,84,141,60,68,52, 81,99,81,113,103,108,109,105,
120,111,126,115,133,122,111,89, 67,99,81,110,116,108,120,90, 90,163,89,121,125,100,107,119, 103,94,88,124,116,114,122,112,
102,136,101,31,69,99,109,94, 95,151,120,85,100,94,78,59, 51,53,85,107,69,157,101,38, 64,67,59,99,103,108,95,116,
79,150,19,27,24,119,22,24, 34,118,44,32,28,19,11,155, 18,43,0,29,14,101,25,9, 34,44,25,79,93,78,75,98,
109,79,94,83,94,67,94,111, 63,103,71,67,102,112,111,62, 121,102,109,118,80,89,83,60, 119,63,60,126,109,130,110,127,
52,53,108,90,118,45,88,105, 66,98,128,92,109,106,132,29, 112,78,109,93,96,54,103,149, 131,47,99,57,55,61,53,92,
113,67,91,86,91,74,66,59, 71,104,115,78,126,131,125,92, 113,78,81,96,142,98,101,64, 76,66,103,108,98,123,99,112,
107,81,112,47,95,153,81,88, 126,155,117,100,111,89,104,77, 131,53,97,95,109,142,93,38, 90,63,95,109,89,98,95,108,
53,65,106,102,111,70,124,115, 115,98,121,126,113,107,115,63, 119,40,128,113,105,54,117,78, 77,126,127,58,62,74,59,98,
130,70,67,61,62,63,68,65, 77,75,76,63,59,56,60,58, 74,89,44,52,52,89,72,70, 112,74,65,152,122,141,139,137,
87,83,128,38,121,100,104,126, 80,90,140,138,111,106,105,56, 118,38,125,105,107,90,118,59, 57,141,42,74,97,86,75,106,
113,89,93,86,69,80,73,66, 73,107,89,65,79,88,93,67, 86,81,65,83,79,83,81,62, 108,93,80,129,114,126,119,125,
72,100,140,106,126,76,78,82, 98,91,119,103,121,124,113,68, 119,69,92,127,109,58,110,23, 82,57,114,116,85,81,78,92,
114,71,89,89,85,92,85,77, 88,84,100,100,109,99,87,42, 81,78,116,95,146,76,98,54, 89,93,94,119,114,113,115,110,
73,57,129,122,104,91,116,112, 130,80,66,119,115,115,115,33, 101,33,124,105,107,25,97,34, 70,127,127,84,81,81,57,94,
74,41,117,137,107,32,76,46, 113,50,108,113,93,131,86,33, 121,33,104,108,103,103,115,9, 34,85,117,79,74,86,80,101,
105,136,69,50,52,146,69,59, 68,160,143,98,90,95,36,91, 54,64,94,68,86,132,126,39, 89,121,50,110,105,109,106,138,
119,64,72,56,60,96,83,58, 74,75,59,61,65,63,48,59, 78,73,49,68,62,65,54,71, 69,57,57,111,119,111,115,117,
};

169
app/src/main/jni/compact_enc_det/compact_enc_det_hint_code.cc
Unescape View File

@ -0,0 +1,169 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#include "compact_enc_det/compact_enc_det_hint_code.h"
#include <ctype.h> // for isalpha
#include <string.h> // for NULL, memchr, strlen, etc
#include "util/basictypes.h" // for uint8, uint32
#include "util/string_util.h"
// Upper to lower, keep digits, everything else to minus '-' (2d)
static const char kCharsetToLowerTbl[256] = {
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
};
static const char kIsAlpha[256] = {
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
};
static const char kIsDigit[256] = {
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
};
static const char* kFakeEncodingName[] = {
"FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
"FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
"FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
"FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
};
static const char* kFakeEncodingName2[] = {
"FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
};
// Return name for extended encoding
const char* MyEncodingName(Encoding enc) {
if (enc < 0) {
return "~";
}
if (enc == ISO_8859_1) {
return "Latin1"; // I can't stand "ASCII" for this
}
if (enc < NUM_ENCODINGS) {
return EncodingName(enc);
}
// allow fake names, for exploration
if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
return kFakeEncodingName2[enc - NUM_ENCODINGS];
}
if ((100 <= enc) && (enc < 120)) {
return kFakeEncodingName[enc - 100];
}
return "~";
}
// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
// Letters are forced to lowercase ASCII
// Used to normalize charset= values
string MakeChar44(const string& str) {
string res("________"); // eight underscores
int l_ptr = 0;
int d_ptr = 0;
for (uint32 i = 0; i < str.size(); ++i) {
uint8 uc = static_cast<uint8>(str[i]);
if (kIsAlpha[uc]) {
if (l_ptr < 4) { // Else ignore
res[l_ptr] = kCharsetToLowerTbl[uc];
l_ptr++;
}
} else if (kIsDigit[uc]) {
if (d_ptr < 4) {
res[4 + d_ptr] = kCharsetToLowerTbl[uc];
} else {
// Keep last 4 digits by shifting left
res[4] = res[5];
res[5] = res[6];
res[6] = res[7];
res[7] = kCharsetToLowerTbl[uc];
}
d_ptr++;
} // If neither letter nor digit, drop entirely
}
return res;
}
// Normalize ASCII string to first 8 alphabetic/digit chars
// Letters are forced to lowercase ASCII
// Used to normalize TLD values
string MakeChar4(const string& str) {
string res("____"); // four underscores
int l_ptr = 0;
for (uint32 i = 0; i < str.size(); ++i) {
uint8 uc = static_cast<uint8>(str[i]);
if (kIsAlpha[uc] | kIsDigit[uc]) {
if (l_ptr < 4) { // Else ignore
res[l_ptr] = kCharsetToLowerTbl[uc];
l_ptr++;
}
}
}
return res;
}
// Normalize ASCII string to first 8 alphabetic/digit chars
// Letters are forced to lowercase ASCII
// Used to normalize TLD values
string MakeChar8(const string& str) {
string res("________"); // eight dots
int l_ptr = 0;
for (uint32 i = 0; i < str.size(); ++i) {
uint8 uc = static_cast<uint8>(str[i]);
if (kIsAlpha[uc] | kIsDigit[uc]) {
if (l_ptr < 8) { // Else ignore
res[l_ptr] = kCharsetToLowerTbl[uc];
l_ptr++;
}
}
}
return res;
}

45
app/src/main/jni/compact_enc_det/compact_enc_det_hint_code.h
Unescape View File

@ -0,0 +1,45 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
#define COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
#include <string> // for string
#include "util/basictypes.h" // for uint32
#include "util/encodings/encodings.h" // for Encoding
using std::string;
// Return name for extended encoding
const char* MyEncodingName(Encoding enc);
// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
// Letters are forced to lowercase ASCII
// Used to normalize charset= values
string MakeChar44(const string& str);
// Normalize ASCII string to first 4 alphabetic/digit chars
// Letters are forced to lowercase ASCII
// Used to normalize TLD values
string MakeChar4(const string& str);
// Normalize ASCII string to first 8 alphabetic/digit chars
// Letters are forced to lowercase ASCII
// Used to normalize other values
string MakeChar8(const string& str);
#endif // COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_

5260
app/src/main/jni/compact_enc_det/compact_enc_det_unittest.cc
View File

File diff suppressed because it is too large Load Diff

152
app/src/main/jni/compact_enc_det/detail_head_string.inc
Unescape View File

@ -0,0 +1,152 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
// Produced by stringify.cc on 2007-09-28 09:13 from file i18n/encodings/compact_enc_det/tools/detail_head.ps
"%!PS-Adobe-2.0\n\n/inch {72 mul} def\n/cshow {dup stringwidth pop -2 div"
" 0 rmoveto show} def\n\n/lmargin 0.5 inch def\n/rmargin 8.5 inch def\n/t"
"margin 10.5 inch def\n/bmargin 0.5 inch def\n\n\n% set to N>=0 to track "
"ranked encoding N\n/track-me -1 def\n/track-me2 -1 def\n\n/columns 2 def"
"\n\n/lpi 18 def % lines per inch\n/lpc lpi 10 mul def "
" % lines per column\n/lpp lpc columns mul def % lines per page\n"
"/probw 3.0 inch def % probability width\n/probr 50 def "
" % probability range\n/widowlines lpi 2 idiv def % 1/2 inch widow a"
"t bottom\n/widowlines lpi def % 1 inch widow at bottom\n\n/lpg l"
"pi 2 idiv def % 1/2 inch spacing between groups\n\n/delc 4 inch "
"def\n/dell -1 inch lpi div def\n\n/next-line 0 def % 24 lines per i"
"nch, 240 per column\n\n/Cfont /Courier findfont 6 scalefont def\n/Hfont "
"/Helvetica findfont 6 scalefont def\nHfont setfont\n\n\n% simple string "
"hash -- sum the characters\n/strhash {\n /hstr exch def\n /h 0 def\n "
"0 1 hstr length 1 sub {/i exch def /h h hstr i get add def} for\n h\n}"
" def\n\n% convert pro at 30 per 2x to 0-2.5 inches spanning -50 to 0\n/p"
"rob2x {\n 30 idiv probr div probw mul neg probw add\n}def\n\n\n/cliptoc"
"olumn {\n % ====== MUST MATCH ME ======\n gsave\n lmargin tmargin mov"
"eto\n next-line lpc idiv delc mul 0 rmoveto\n -1 18 rmoveto 0 -10.5 in"
"ch rlineto delc 2 add 0 rlineto 0 10.5 inch rlineto closepath\n clip\n"
" newpath\n} def\n\n/endcliptocolumn {\n grestore\n % ====== MUST MATC"
"H ME ======\n} def\n\n\n\n/column-box {\n lmargin tmargin moveto\n nex"
"t-line 1 sub lpc idiv delc mul next-line 1 sub lpc mod 1 add dell mul r"
"moveto\n % box\n gsave -1.5 0 rmoveto 0 detail-count dell mul neg rmov"
"eto probw 3 add 0 rlineto 0.25 setlinewidth stroke grestore\n gsave -1"
".5 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth strok"
"e grestore\n gsave probw .8 mul 0 rmoveto 0 detail-count dell mul neg r"
"lineto 0.25 setlinewidth 0.8 setgray stroke grestore\n gsave probw .6 m"
"ul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth 0.8 s"
"etgray stroke grestore\n gsave probw .4 mul 0 rmoveto 0 detail-count de"
"ll mul neg rlineto 0.25 setlinewidth 0.8 setgray stroke grestore\n gsav"
"e probw .2 mul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlin"
"ewidth 0.8 setgray stroke grestore\n gsave probw 1.5 add 0 rmoveto 0 de"
"tail-count dell mul neg rlineto 0.25 setlinewidth stroke grestore\n} def"
"\n\n\n/IncrementLine {\n /incr exch def\n /next-line next-line incr ad"
"d def\n next-line lpc ge next-line incr sub lpc lt and {\n % We "
"just went to the top of column 2; redo clip\n endcliptocolumn % M"
"UST match\n column-box\n /next-line lpc def\n cliptocolumn "
" % MUST match\n } if\n next-line lpp ge {\n % We just went to the"
" top of column 3; start new page column 1\n endcliptocolumn % MUS"
"T match\n column-box\n showpage\n Hfont setfont\n /next-line"
" 0 def\n show-pageno\n cliptocolumn % MUST match\n } if\n}"
" def\n\n/IncrementLineOutside {\n /incr exch def\n /next-line next-lin"
"e incr add def\n next-line lpc ge next-line incr sub lpc lt and {\n"
" % We just went to the top of column 2\n /next-line lpc def\n } i"
"f\n next-line lpp ge {\n % We just went to the top of column 3; star"
"t new page column 1\n showpage\n Hfont setfont\n /next-line 0 d"
"ef\n show-pageno\n } if\n} def\n\n/NextColumn {\n lpc 1 sub Increme"
"ntLine\n} def\n\n/NextPage {\n lpp 1 sub IncrementLine\n} def\n\n% Up"
"on entry, we are OUTSIDE the clip\n/start-detail {\n /d-title exch def\n"
"\n % align >= 1 inch at bottom of column, and/or start new column\n lp"
"c next-line lpc mod sub widowlines lt {\n % Start at top of a column\n"
" next-line lpc ge {\n % Start new page\n showpage\n Hf"
"ont setfont\n /next-line 0 def\n show-pageno\n } {\n %"
" Start new column\n /next-line lpc def\n } ifelse\n } if\n\n l"
"margin tmargin moveto\n next-line lpc idiv delc mul next-line lpc mod "
"dell mul rmoveto\n gsave d-title show grestore\n 0 dell rmoveto\n 1 1"
" 4 {/j exch def gsave probw j mul 5 div -2 rmoveto 50 j 10 mul sub 20 st"
"ring cvs cshow grestore} for\n 2 IncrementLineOutside\n /detail-count "
"1 def\n cliptocolumn % MUST match\n /d-array [] def\n} def\n"
"\n/size-detail {\n /d-names exch def\n /d-size exch def\n % zero sums"
"\n /sums d-size array def\n 0 1 d-size 1 sub {/i exch def sums i 0 pu"
"t} for\n /old-d-max 0 def\n /colors d-size array def\n 0 1 d-size 1 s"
"ub {/i exch def colors i i 3 mul 17 mod 17 div put} for\n %0 1 d-size "
"1 sub {/i exch def colors i d-names i get strhash 3 mul 17 mod 17 div p"
"ut} for\n %0 1 d-size 1 sub {/i exch def ( ) show colors i get 20 stri"
"ng cvs show} for\n} def\n\n/count-detail {\n /detail-total-count exch d"
"ef\n % if total-count >= one column, start at top of a column\n detail"
"-total-count lpp ge {\n % Start new page\n NextPage\n } {\n de"
"tail-total-count lpc ge {\n % Start new column\n NextColumn\n "
" } if\n } ifelse\n} def\n\n% highlight next entry with underbar\n/do-"
"flag {\ngsave\n setrgbcolor\n lmargin tmargin moveto\n next-line lpc "
"idiv delc mul next-line lpc mod dell mul rmoveto\n 0 -2 rmoveto\n pro"
"bw 0 rlineto\n 0 dell neg rlineto\n probw neg 0 rlineto\n closepath\n"
" fill\ngrestore\n} def\n\n/do-detail-e {\n /d-array exch def\n /d-enc"
" exch def\n /d-label exch def\n /d-max -999999 def\n\n lmargin tmargi"
"n moveto\n next-line lpc idiv delc mul next-line lpc mod dell mul rmov"
"eto\n 0.25 setlinewidth\n\n % show label, using encoding color\n gsav"
"e\n probw 2 add -2 rmoveto\n detail-count 1 sub 2 mod 0.25 inch mu"
"l 0 rmoveto\n % ([) show detail-count 20 string cvs show (] ) show\n "
" d-enc 0 lt {\n 0 setgray\n }{\n colors d-enc get 1 .8 se"
"thsbcolor\n } ifelse\n d-label show\n grestore\n % For -prune- d"
"raw horizontal line\n d-label length 8 gt {d-label 4 get (p) 0 get eq d"
"-label 5 get (r) 0 get eq and {\n /prune-val d-label cvi def\n /ne"
"wx prune-val 30 mul prob2x def\n gsave newx 6 rmoveto 0 -12 rlineto 1"
".5 setlinewidth 0.8 setgray stroke grestore\n gsave probw 0 add 0 rli"
"neto 0.25 setlinewidth 0.8 setgray stroke grestore\n } if } if\n\n % t"
"rack max per new row\n 0 1 d-array length 1 sub {\n /i exch def\n "
" /sum sums i get d-array i get add def\n d-max sum lt {/d-max sum def"
"} if\n } for\n\n % draw line increments\n 0 1 d-array length 1 sub {\n"
" /i exch def\n detail-count 1 gt {\n /oldx old-d-max sums i g"
"et sub prob2x def\n } {\n /oldx 600 prob2x def\n } ifelse\n "
" /oldy dell neg def\n /newx d-max sums i get d-array i get add sub"
" prob2x def\n /newy 0 def\n gsave\n oldx oldy rmoveto\n newx"
" oldx sub newy oldy sub rlineto\n % if encoding is being tracked, ma"
"ke bold line\n i track-me eq\n {2 setlinewidth}\n {i track-"
"me2 eq {1.25 setlinewidth} {0.25 setlinewidth} ifelse}\n ifelse\n "
" colors i get 1 .8 sethsbcolor stroke\n grestore\n } for\n /detail"
"-count detail-count 1 add def\n\n % increment running total in sums, tr"
"ack max per row\n 0 1 d-array length 1 sub {\n /i exch def\n sums"
" i sums i get d-array i get add put\n } for\n /old-d-max d-max def\n"
"\n 1 IncrementLine\n} def\n\n\n/do-detail {\n /d-array exch def\n /d-"
"label exch def\n d-label -1 d-array do-detail-e\n} def\n\n% Upon exit, "
"we are outside the clip\n/end-detail {\n pop\n endcliptocolumn "
" % MUST match\n column-box\n\n % text labels\n 0 1 d-array length 1 s"
"ub {\n /i exch def\n gsave\n /newx old-d-max sums i get sub pro"
"b2x def\n newx 0 ge {\n newx 0 rmoveto\n currentpoint trans"
"late\n colors i get 1 .8 sethsbcolor\n gsave 0 dell neg rline"
"to 0.25 setlinewidth stroke grestore\n -60 rotate\n 0 -2 movet"
"o d-names i get show\n } if\n grestore\n } for\n d-array length "
"0 gt {\n lpg IncrementLineOutside\n } {\n lpg 4 idiv IncrementLin"
"eOutside\n } ifelse\n} def\n\n/do-src {\n/src exch def\n lmargin tmarg"
"in moveto\n next-line lpc idiv delc mul next-line lpc mod dell mul rmo"
"veto\n Cfont setfont\n src show\n Hfont setfont\n 1 IncrementLine\n}"
" def\n\n% Underline trigram in source text\n/do-highlight1 {\n /hl-colo"
"r exch def\n /hl-offset exch def\n /hl-line exch 1 sub 2 mul def\n gs"
"ave\n lmargin tmargin moveto\n next-line hl-line sub lpc idiv delc mul"
"\n next-line hl-line sub lpc mod dell mul rmoveto\n % Assume text is 6"
" chars in and 3.6 pts per char, but 2 chars per offset\n hl-offset 2 mu"
"l 6 add 3.6 mul 4 rmoveto\n\n 0 setgray 0.5 setlinewidth\n hl-color 1"
" eq {0 0 1 setrgbcolor} if % Latin1 blue\n hl-color 2 eq {1 0 1 setrgb"
"color} if % Latin2 magenta\n hl-color 3 eq {1 0.67 0 setrgbcolor} if "
"% Latin7 orange\n 18 -2 rlineto stroke\n grestore\n} def\n\n% Box trig"
"ram in source text\n/do-highlight2 {\n /hl-color exch def\n /hl-offset"
" exch def\n /hl-line exch 1 sub 2 mul def\n gsave\n lmargin tmargin m"
"oveto\n next-line hl-line sub lpc idiv delc mul\n next-line hl-line su"
"b lpc mod dell mul rmoveto\n % Assume text is 6 chars in and 3.6 pts pe"
"r char, but 2 chars per offset\n hl-offset 2 mul 6 add 3.6 mul 4 rmove"
"to\n\n 0 setgray 0.25 setlinewidth\n hl-color 1 eq {0 0 1 setrgbcolor}"
" if % Latin1 blue\n hl-color 2 eq {1 0 1 setrgbcolor} if % Latin2 mag"
"enta\n hl-color 3 eq {1 0.67 0 setrgbcolor} if % Latin7 orange\n -0.5"
" -0.5 rmoveto\n 22 0 rlineto\n 0 4 rlineto\n -11 2 rlineto\n -11 -2 "
"rlineto\n closepath\n stroke\n grestore\n} def\n\n/show-pageno {\ngsa"
"ve\nlmargin bmargin moveto 0 -12 rmoveto\n(Page ) show pageno 20 string "
"cvs show\ngrestore\n/pageno pageno 1 add def\n} def\n\n/pageno 1 def\nsh"
"ow-pageno\n%=============================\n\n\n"

331
app/src/main/jni/util/basictypes.h
Unescape View File

@ -0,0 +1,331 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_BASICTYPES_H_
#define UTIL_BASICTYPES_H_
#include <limits.h> // So we can set the bounds of our types
#include <stddef.h> // For size_t
#include <string.h> // for memcpy
#include "util/port.h" // Types that only need exist on certain systems
#ifndef COMPILER_MSVC
// stdint.h is part of C99 but MSVC doesn't have it.
#include <stdint.h> // For intptr_t.
#endif
typedef signed char schar;
typedef signed char int8;
typedef short int16;
// TODO(mbelshe) Remove these type guards. These are
// temporary to avoid conflicts with npapi.h.
#ifndef _INT32
#define _INT32
typedef int int32;
#endif
// The NSPR system headers define 64-bit as |long| when possible. In order to
// not have typedef mismatches, we do the same on LP64.
#if __LP64__
typedef long int64;
#else
typedef long long int64;
#endif
// NOTE: unsigned types are DANGEROUS in loops and other arithmetical
// places. Use the signed types unless your variable represents a bit
// pattern (eg a hash value) or you really need the extra bit. Do NOT
// use 'unsigned' to express "this value should always be positive";
// use assertions for this.
typedef unsigned char uint8;
typedef unsigned short uint16;
// TODO(mbelshe) Remove these type guards. These are
// temporary to avoid conflicts with npapi.h.
#ifndef _UINT32
#define _UINT32
typedef unsigned int uint32;
#endif
// See the comment above about NSPR and 64-bit.
#if __LP64__
typedef unsigned long uint64;
#else
typedef unsigned long long uint64;
#endif
// A type to represent a Unicode code-point value. As of Unicode 4.0,
// such values require up to 21 bits.
// (For type-checking on pointers, make this explicitly signed,
// and it should always be the signed version of whatever int32 is.)
typedef signed int char32;
const uint8 kuint8max = (( uint8) 0xFF);
const uint16 kuint16max = ((uint16) 0xFFFF);
const uint32 kuint32max = ((uint32) 0xFFFFFFFF);
const uint64 kuint64max = ((uint64) GG_LONGLONG(0xFFFFFFFFFFFFFFFF));
const int8 kint8min = (( int8) 0x80);
const int8 kint8max = (( int8) 0x7F);
const int16 kint16min = (( int16) 0x8000);
const int16 kint16max = (( int16) 0x7FFF);
const int32 kint32min = (( int32) 0x80000000);
const int32 kint32max = (( int32) 0x7FFFFFFF);
const int64 kint64min = (( int64) GG_LONGLONG(0x8000000000000000));
const int64 kint64max = (( int64) GG_LONGLONG(0x7FFFFFFFFFFFFFFF));
// A macro to disallow the copy constructor and operator= functions
// This should be used in the private: declarations for a class
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName&); \
void operator=(const TypeName&)
// An older, deprecated, politically incorrect name for the above.
#define DISALLOW_EVIL_CONSTRUCTORS(TypeName) DISALLOW_COPY_AND_ASSIGN(TypeName)
// A macro to disallow all the implicit constructors, namely the
// default constructor, copy constructor and operator= functions.
//
// This should be used in the private: declarations for a class
// that wants to prevent anyone from instantiating it. This is
// especially useful for classes containing only static methods.
#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
TypeName(); \
DISALLOW_COPY_AND_ASSIGN(TypeName)
// The arraysize(arr) macro returns the # of elements in an array arr.
// The expression is a compile-time constant, and therefore can be
// used in defining new arrays, for example. If you use arraysize on
// a pointer by mistake, you will get a compile-time error.
// This template function declaration is used in defining arraysize.
// Note that the function doesn't need an implementation, as we only
// use its type.
template <typename T, size_t N>
char (&ArraySizeHelper(T (&array)[N]))[N];
// That gcc wants both of these prototypes seems mysterious. VC, for
// its part, can't decide which to use (another mystery). Matching of
// template overloads: the final frontier.
#ifndef _MSC_VER
template <typename T, size_t N>
char (&ArraySizeHelper(const T (&array)[N]))[N];
#endif
#define arraysize(array) (sizeof(ArraySizeHelper(array)))
// Use implicit_cast as a safe version of static_cast or const_cast
// for upcasting in the type hierarchy (i.e. casting a pointer to Foo
// to a pointer to SuperclassOfFoo or casting a pointer to Foo to
// a const pointer to Foo).
// When you use implicit_cast, the compiler checks that the cast is safe.
// Such explicit implicit_casts are necessary in surprisingly many
// situations where C++ demands an exact type match instead of an
// argument type convertable to a target type.
//
// The From type can be inferred, so the preferred syntax for using
// implicit_cast is the same as for static_cast etc.:
//
// implicit_cast<ToType>(expr)
//
// implicit_cast would have been part of the C++ standard library,
// but the proposal was submitted too late. It will probably make
// its way into the language in the future.
template<typename To, typename From>
inline To implicit_cast(From const &f) {
return f;
}
// The COMPILE_ASSERT macro can be used to verify that a compile time
// expression is true. For example, you could use it to verify the
// size of a static array:
//
// COMPILE_ASSERT(arraysize(content_type_names) == CONTENT_NUM_TYPES,
// content_type_names_incorrect_size);
//
// or to make sure a struct is smaller than a certain size:
//
// COMPILE_ASSERT(sizeof(foo) < 128, foo_too_large);
//
// The second argument to the macro is the name of the variable. If
// the expression is false, most compilers will issue a warning/error
// containing the name of the variable.
template <bool>
struct CompileAssert {
};
#undef COMPILE_ASSERT
#define COMPILE_ASSERT(expr, msg) \
typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1]
// Implementation details of COMPILE_ASSERT:
//
// - COMPILE_ASSERT works by defining an array type that has -1
// elements (and thus is invalid) when the expression is false.
//
// - The simpler definition
//
// #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1]
//
// does not work, as gcc supports variable-length arrays whose sizes
// are determined at run-time (this is gcc's extension and not part
// of the C++ standard). As a result, gcc fails to reject the
// following code with the simple definition:
//
// int foo;
// COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is
// // not a compile-time constant.
//
// - By using the type CompileAssert<(bool(expr))>, we ensures that
// expr is a compile-time constant. (Template arguments must be
// determined at compile-time.)
//
// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written
//
// CompileAssert<bool(expr)>
//
// instead, these compilers will refuse to compile
//
// COMPILE_ASSERT(5 > 0, some_message);
//
// (They seem to think the ">" in "5 > 0" marks the end of the
// template argument list.)
//
// - The array size is (bool(expr) ? 1 : -1), instead of simply
//
// ((expr) ? 1 : -1).
//
// This is to avoid running into a bug in MS VC 7.1, which
// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
// MetatagId refers to metatag-id that we assign to
// each metatag <name, value> pair..
typedef uint32 MetatagId;
// Argument type used in interfaces that can optionally take ownership
// of a passed in argument. If TAKE_OWNERSHIP is passed, the called
// object takes ownership of the argument. Otherwise it does not.
enum Ownership {
DO_NOT_TAKE_OWNERSHIP,
TAKE_OWNERSHIP
};
// bit_cast<Dest,Source> is a template function that implements the
// equivalent of "*reinterpret_cast<Dest*>(&source)". We need this in
// very low-level functions like the protobuf library and fast math
// support.
//
// float f = 3.14159265358979;
// int i = bit_cast<int32>(f);
// // i = 0x40490fdb
//
// The classical address-casting method is:
//
// // WRONG
// float f = 3.14159265358979; // WRONG
// int i = * reinterpret_cast<int*>(&f); // WRONG
//
// The address-casting method actually produces undefined behavior
// according to ISO C++ specification section 3.10 -15 -. Roughly, this
// section says: if an object in memory has one type, and a program
// accesses it with a different type, then the result is undefined
// behavior for most values of "different type".
//
// This is true for any cast syntax, either *(int*)&f or
// *reinterpret_cast<int*>(&f). And it is particularly true for
// conversions betweeen integral lvalues and floating-point lvalues.
//
// The purpose of 3.10 -15- is to allow optimizing compilers to assume
// that expressions with different types refer to different memory. gcc
// 4.0.1 has an optimizer that takes advantage of this. So a
// non-conforming program quietly produces wildly incorrect output.
//
// The problem is not the use of reinterpret_cast. The problem is type
// punning: holding an object in memory of one type and reading its bits
// back using a different type.
//
// The C++ standard is more subtle and complex than this, but that
// is the basic idea.
//
// Anyways ...
//
// bit_cast<> calls memcpy() which is blessed by the standard,
// especially by the example in section 3.9 . Also, of course,
// bit_cast<> wraps up the nasty logic in one place.
//
// Fortunately memcpy() is very fast. In optimized mode, with a
// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
// code with the minimal amount of data movement. On a 32-bit system,
// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
// compiles to two loads and two stores.
//
// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
//
// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
// is likely to surprise you.
template <class Dest, class Source>
inline Dest bit_cast(const Source& source) {
// Compile time assertion: sizeof(Dest) == sizeof(Source)
// A compile error here means your Dest and Source have different sizes.
// typedef char VerifySizesAreEqual [sizeof(Dest) == sizeof(Source) ? 1 : -1];
Dest dest;
memcpy(&dest, &source, sizeof(dest));
return dest;
}
// The following enum should be used only as a constructor argument to indicate
// that the variable has static storage class, and that the constructor should
// do nothing to its state. It indicates to the reader that it is legal to
// declare a static instance of the class, provided the constructor is given
// the base::LINKER_INITIALIZED argument. Normally, it is unsafe to declare a
// static variable that has a constructor or a destructor because invocation
// order is undefined. However, IF the type can be initialized by filling with
// zeroes (which the loader does for static variables), AND the destructor also
// does nothing to the storage, AND there are no virtual methods, then a
// constructor declared as
// explicit MyClass(base::LinkerInitialized x) {}
// and invoked as
// static MyClass my_variable_name(base::LINKER_INITIALIZED);
namespace base {
enum LinkerInitialized { LINKER_INITIALIZED };
} // base
// UnaligndLoad32 is put here instead of util/port.h to
// avoid the circular dependency between port.h and basictypes.h
// ARM does not support unaligned memory access.
#if defined(ARCH_CPU_X86_FAMILY)
// x86 and x86-64 can perform unaligned loads/stores directly;
inline uint32 UnalignedLoad32(const void* p) {
return *reinterpret_cast<const uint32*>(p);
}
#else
#define NEED_ALIGNED_LOADS
// If target architecture does not support unaligned loads and stores,
// use memcpy version of UNALIGNED_LOAD32.
inline uint32 UnalignedLoad32(const void* p) {
uint32 t;
memcpy(&t, reinterpret_cast<const uint8*>(p), sizeof(t));
return t;
}
#endif
#endif // UTIL_BASICTYPES_H_

88
app/src/main/jni/util/case_insensitive_hash.h
Unescape View File

@ -0,0 +1,88 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_CASE_INSENSITIVE_HASH_H_
#define UTIL_CASE_INSENSITIVE_HASH_H_
#include <ctype.h>
#include <stddef.h>
#ifndef _MSC_VER
#include <strings.h>
#endif
#include <string>
#include "util/basictypes.h"
#include "util/string_util.h"
// Functors for hashing c-strings with case-insensitive semantics.
struct CStringCaseHash {
size_t operator()(const char *str) const {
unsigned long hash_val = 0;
while (*str) {
hash_val = 5*hash_val + tolower(*str);
str++;
}
return (size_t)hash_val;
}
};
struct CStringCaseEqual {
bool operator()(const char *str1, const char *str2) const {
return !base::strcasecmp(str1, str2);
}
};
// These functors, in addition to being case-insensitive, ignore all
// non-alphanumeric characters. This is useful when we want all variants of
// a string -- where variants can differ in puncutation and whitespace -- to
// map to the same value.
struct CStringAlnumCaseHash {
size_t operator()(const char *str) const {
unsigned long hash_val = 0;
while (*str) {
if (isalnum(*str)) {
hash_val = 5*hash_val + tolower(*str);
}
str++;
}
return (size_t)hash_val;
}
};
struct CStringAlnumCaseEqual {
bool operator()(const char *str1, const char *str2) const {
while (true) {
// Skip until each pointer is pointing to an alphanumeric char or '\0'
while (!isalnum(*str1) && (*str1 != '\0')) {
str1++;
}
while (!isalnum(*str2) && (*str2 != '\0')) {
str2++;
}
if (tolower(*str1) != tolower(*str2)) {
return false; // mismatch on alphanumeric char or '\0'
}
if (*str1 == '\0') { // in which case *str2 must be '\0' as well
return true; // reached '\0' in both strings without mismatch
}
str1++;
str2++;
}
}
};
#endif // UTIL_CASE_INSENSITIVE_HASH_H_

39
app/src/main/jni/util/commandlineflags.h
Unescape View File

@ -0,0 +1,39 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_COMMANDLINEFLAGS_H_
#define UTIL_COMMANDLINEFLAGS_H_
#undef DEFINE_bool
#define DEFINE_bool(name, default_value, comment) \
bool FLAGS_##name = default_value
#undef DEFINE_int32
#define DEFINE_int32(name, default_value, comment) \
int32 FLAGS_##name = default_value
#undef DEFINE_string
#define DEFINE_string(name, default_value, comment) \
string FLAGS_##name = default_value
#undef DECLARE_bool
#define DECLARE_bool(name) extern bool FLAGS_##name
#undef DECLARE_int32
#define DECLARE_int32(name) extern int32 FLAGS_##name
#undef DECLARE_string
#define DECLARE_string(name) extern string FLAGS_##name
#endif // UTIL_COMMANDLINEFLAGS_H_

891
app/src/main/jni/util/encodings/encodings.cc
Unescape View File

@ -0,0 +1,891 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#include "util/encodings/encodings.h"
#include <string.h> // for strcasecmp
#include <unordered_map>
#include <utility> // for pair
#include "util/basictypes.h"
#include "util/string_util.h"
#include "util/case_insensitive_hash.h"
struct EncodingInfo {
// The standard name for this encoding.
//
const char* encoding_name_;
// The "preferred MIME name" of an encoding as specified by the IANA at:
// http://www.iana.org/assignments/character-sets
//
// Note that the preferred MIME name may differ slightly from the
// official IANA name: i.e. ISO-8859-1 vs. ISO_8859-1:1987
//
const char* mime_encoding_name_;
// It is an internal policy that if an encoding has an IANA name,
// then encoding_name_ and mime_encoding_name_ must be the same string.
//
// However, there can be exceptions if there are compelling reasons.
// For example, Japanese mobile handsets require the name
// "Shift_JIS" in charset=... parameter in Content-Type headers to
// process emoji (emoticons) in their private encodings. In that
// case, mime_encoding_name_ should be "Shift_JIS", despite
// encoding_name_ actually is "X-KDDI-Shift_JIS".
// Some multi-byte encodings use byte values that coincide with the
// ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
// can misinterpret these, as indicated in an external XSS report from
// 2007-02-15. Here, we map these dangerous encodings to safer ones. We
// also use UTF8 instead of encodings that we don't support in our
// output, and we generally try to be conservative in what we send out.
// Where the client asks for single- or double-byte encodings that are
// not as common, we substitute a more common single- or double-byte
// encoding, if there is one, thereby preserving the client's intent
// to use less space than UTF-8. This also means that characters
// outside the destination set will be converted to HTML NCRs (&#NNN;)
// if requested.
Encoding preferred_web_output_encoding_;
};
static const EncodingInfo kEncodingInfoTable[] = {
{ "ASCII", "ISO-8859-1", ISO_8859_1},
{ "Latin2", "ISO-8859-2", ISO_8859_2},
{ "Latin3", "ISO-8859-3", UTF8},
// MSIE 6 does not support ISO-8859-3 (XSS issue)
{ "Latin4", "ISO-8859-4", ISO_8859_4},
{ "ISO-8859-5", "ISO-8859-5", ISO_8859_5},
{ "Arabic", "ISO-8859-6", ISO_8859_6},
{ "Greek", "ISO-8859-7", ISO_8859_7},
{ "Hebrew", "ISO-8859-8", MSFT_CP1255},
// we do not endorse the visual order
{ "Latin5", "ISO-8859-9", ISO_8859_9},
{ "Latin6", "ISO-8859-10", UTF8},
// MSIE does not support ISO-8859-10 (XSS issue)
{ "EUC-JP", "EUC-JP", JAPANESE_EUC_JP},
{ "SJS", "Shift_JIS", JAPANESE_SHIFT_JIS},
{ "JIS", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
// due to potential confusion with HTML syntax chars
{ "BIG5", "Big5", CHINESE_BIG5},
{ "GB", "GB2312", CHINESE_GB},
{ "EUC-CN",
"EUC-CN",
// Misnamed. Should be EUC-TW.
CHINESE_BIG5},
// MSIE treats "EUC-CN" like GB2312, which is not EUC-TW,
// and EUC-TW is rare, so we prefer Big5 for output.
{ "KSC", "EUC-KR", KOREAN_EUC_KR},
{ "Unicode",
"UTF-16LE",
// Internet Explorer doesn't recognize "ISO-10646-UCS-2"
UTF8
// due to potential confusion with HTML syntax chars
},
{ "EUC",
"EUC", // Misnamed. Should be EUC-TW.
CHINESE_BIG5
// MSIE does not recognize "EUC" (XSS issue),
// and EUC-TW is rare, so we prefer Big5 for output.
},
{ "CNS",
"CNS", // Misnamed. Should be EUC-TW.
CHINESE_BIG5},
// MSIE does not recognize "CNS" (XSS issue),
// and EUC-TW is rare, so we prefer Big5 for output.
{ "BIG5-CP950",
"BIG5-CP950", // Not an IANA name
CHINESE_BIG5
// MSIE does not recognize "BIG5-CP950" (XSS issue)
},
{ "CP932", "CP932", // Not an IANA name
JAPANESE_SHIFT_JIS}, // MSIE does not recognize "CP932" (XSS issue)
{ "UTF8", "UTF-8", UTF8},
{ "Unknown",
"x-unknown", // Not an IANA name
UTF8}, // UTF-8 is our default output encoding
{ "ASCII-7-bit", "US-ASCII", ASCII_7BIT},
{ "KOI8R", "KOI8-R", RUSSIAN_KOI8_R},
{ "CP1251", "windows-1251", RUSSIAN_CP1251},
{ "CP1252", "windows-1252", MSFT_CP1252},
{ "KOI8U",
"KOI8-U",
ISO_8859_5}, // because koi8-u is not as common
{ "CP1250", "windows-1250", MSFT_CP1250},
{ "ISO-8859-15", "ISO-8859-15", ISO_8859_15},
{ "CP1254", "windows-1254", MSFT_CP1254},
{ "CP1257", "windows-1257", MSFT_CP1257},
{ "ISO-8859-11", "ISO-8859-11", ISO_8859_11},
{ "CP874", "windows-874", MSFT_CP874},
{ "CP1256", "windows-1256", MSFT_CP1256},
{ "CP1255", "windows-1255", MSFT_CP1255},
{ "ISO-8859-8-I", "ISO-8859-8-I", MSFT_CP1255},
// Java does not support iso-8859-8-i
{ "VISUAL", "ISO-8859-8", MSFT_CP1255},
// we do not endorse the visual order
{ "CP852", "cp852", MSFT_CP1250},
// because cp852 is not as common
{ "CSN_369103", "csn_369103", MSFT_CP1250},
// MSIE does not recognize "csn_369103" (XSS issue)
{ "CP1253", "windows-1253", MSFT_CP1253},
{ "CP866", "IBM866", RUSSIAN_CP1251},
// because cp866 is not as common
{ "ISO-8859-13", "ISO-8859-13", UTF8},
// because iso-8859-13 is not widely supported
{ "ISO-2022-KR", "ISO-2022-KR", KOREAN_EUC_KR},
// due to potential confusion with HTML syntax chars
{ "GBK", "GBK", GBK},
{ "GB18030", "GB18030", GBK},
// because gb18030 is not widely supported
{ "BIG5_HKSCS", "BIG5-HKSCS", CHINESE_BIG5},
// because Big5-HKSCS is not widely supported
{ "ISO_2022_CN", "ISO-2022-CN", CHINESE_GB},
// due to potential confusion with HTML syntax chars
{ "TSCII", "tscii", UTF8},
// we do not have an output converter for this font encoding
{ "TAM", "tam", UTF8},
// we do not have an output converter for this font encoding
{ "TAB", "tab", UTF8},
// we do not have an output converter for this font encoding
{ "JAGRAN", "jagran", UTF8},
// we do not have an output converter for this font encoding
{ "MACINTOSH", "MACINTOSH", ISO_8859_1},
// because macintosh is relatively uncommon
{ "UTF7", "UTF-7",
UTF8}, // UTF-7 has been the subject of XSS attacks and is deprecated
{ "BHASKAR", "bhaskar",
UTF8}, // we do not have an output converter for this font encoding
{ "HTCHANAKYA", "htchanakya", // not an IANA charset name.
UTF8}, // we do not have an output converter for this font encoding
{ "UTF-16BE", "UTF-16BE",
UTF8}, // due to potential confusion with HTML syntax chars
{ "UTF-16LE", "UTF-16LE",
UTF8}, // due to potential confusion with HTML syntax chars
{ "UTF-32BE", "UTF-32BE",
UTF8}, // unlikely to cause XSS bugs, but very uncommon on Web
{ "UTF-32LE", "UTF-32LE",
UTF8}, // unlikely to cause XSS bugs, but very uncommon on Web
{ "X-BINARYENC", "x-binaryenc", // Not an IANA name
UTF8}, // because this one is not intended for output (just input)
{ "HZ-GB-2312", "HZ-GB-2312",
CHINESE_GB}, // due to potential confusion with HTML syntax chars
{ "X-UTF8UTF8", "x-utf8utf8", // Not an IANA name
UTF8}, // because this one is not intended for output (just input)
{ "X-TAM-ELANGO", "x-tam-elango",
UTF8}, // we do not have an output converter for this font encoding
{ "X-TAM-LTTMBARANI", "x-tam-lttmbarani",
UTF8}, // we do not have an output converter for this font encoding
{ "X-TAM-SHREE", "x-tam-shree",
UTF8}, // we do not have an output converter for this font encoding
{ "X-TAM-TBOOMIS", "x-tam-tboomis",
UTF8}, // we do not have an output converter for this font encoding
{ "X-TAM-TMNEWS", "x-tam-tmnews",
UTF8}, // we do not have an output converter for this font encoding
{ "X-TAM-WEBTAMIL", "x-tam-webtamil",
UTF8}, // we do not have an output converter for this font encoding
{ "X-KDDI-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
// KDDI version of Shift_JIS with Google Emoji PUA mappings.
// Note that MimeEncodingName() returns "Shift_JIS", since KDDI uses
// "Shift_JIS" in HTTP headers and email messages.
{ "X-DoCoMo-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
// DoCoMo version of Shift_JIS with Google Emoji PUA mappings.
// See the comment at KDDI_SHIFT_JIS for other issues.
{ "X-SoftBank-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
// SoftBank version of Shift_JIS with Google Emoji PUA mappings.
// See the comment at KDDI_SHIFT_JIS for other issues.
{ "X-KDDI-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
// KDDI version of ISO-2022-JP with Google Emoji PUA mappings.
// See the comment at KDDI_SHIFT_JIS for other issues.
// The preferred Web encoding is due to potential confusion with
// HTML syntax chars.
{ "X-SoftBank-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
// SoftBank version of ISO-2022-JP with Google Emoji PUA mappings.
// See the comment at KDDI_SHIFT_JIS for other issues.
// The preferred Web encoding is due to potential confusion with
// HTML syntax chars.
// Please refer to NOTE: section in the comments in the definition
// of "struct I18NInfoByEncoding", before adding new encodings.
};
COMPILE_ASSERT(arraysize(kEncodingInfoTable) == NUM_ENCODINGS,
kEncodingInfoTable_has_incorrect_size);
Encoding default_encoding() {return LATIN1;}
// *************************************************************
// Encoding predicates
// IsValidEncoding()
// IsEncEncCompatible
// IsEncodingWithSupportedLanguage
// IsSupersetOfAscii7Bit
// Is8BitEncoding
// IsCJKEncoding
// IsHebrewEncoding
// IsRightToLeftEncoding
// IsLogicalRightToLeftEncoding
// IsVisualRightToLeftEncoding
// IsIso2022Encoding
// IsIso2022JpOrVariant
// IsShiftJisOrVariant
// IsJapaneseCellPhoneCarrierSpecificEncoding
// *************************************************************
bool IsValidEncoding(Encoding enc) {
return ((enc >= 0) && (enc < kNumEncodings));
}
bool IsEncEncCompatible(const Encoding from, const Encoding to) {
// Tests compatibility between the "from" and "to" encodings; in
// the typical case -- when both are valid known encodings -- this
// returns true iff converting from first to second is a no-op.
if (!IsValidEncoding(from) || !IsValidEncoding(to)) {
return false; // we only work with valid encodings...
} else if (to == from) {
return true; // the trivial common case
}
if (to == UNKNOWN_ENCODING) {
return true; // all valid encodings are compatible with the unknown
}
if (from == UNKNOWN_ENCODING) {
return false; // no unknown encoding is compatible with one that is
}
if (from == ASCII_7BIT) {
return IsSupersetOfAscii7Bit(to);
}
return (from == ISO_8859_1 && to == MSFT_CP1252) ||
(from == ISO_8859_8 && to == HEBREW_VISUAL) ||
(from == HEBREW_VISUAL && to == ISO_8859_8) ||
(from == ISO_8859_9 && to == MSFT_CP1254) ||
(from == ISO_8859_11 && to == MSFT_CP874) ||
(from == JAPANESE_SHIFT_JIS && to == JAPANESE_CP932) ||
(from == CHINESE_BIG5 && to == CHINESE_BIG5_CP950) ||
(from == CHINESE_GB && to == GBK) ||
(from == CHINESE_GB && to == GB18030) ||
(from == CHINESE_EUC_CN && to == CHINESE_EUC_DEC) ||
(from == CHINESE_EUC_CN && to == CHINESE_CNS) ||
(from == CHINESE_EUC_DEC && to == CHINESE_EUC_CN) ||
(from == CHINESE_EUC_DEC && to == CHINESE_CNS) ||
(from == CHINESE_CNS && to == CHINESE_EUC_CN) ||
(from == CHINESE_CNS && to == CHINESE_EUC_DEC);
}
// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
// encoding represent the same characters as they do in ISO_8859_1.
// TODO: This list could be expanded. Many other encodings are supersets
// of 7-bit Ascii. In fact, Japanese JIS and Unicode are the only two
// encodings that I know for a fact should *not* be in this list.
bool IsSupersetOfAscii7Bit(Encoding e) {
switch (e) {
case ISO_8859_1:
case ISO_8859_2:
case ISO_8859_3:
case ISO_8859_4:
case ISO_8859_5:
case ISO_8859_6:
case ISO_8859_7:
case ISO_8859_8:
case ISO_8859_9:
case ISO_8859_10:
case JAPANESE_EUC_JP:
case JAPANESE_SHIFT_JIS:
case CHINESE_BIG5:
case CHINESE_GB:
case CHINESE_EUC_CN:
case KOREAN_EUC_KR:
case CHINESE_EUC_DEC:
case CHINESE_CNS:
case CHINESE_BIG5_CP950:
case JAPANESE_CP932:
case UTF8:
case UNKNOWN_ENCODING:
case ASCII_7BIT:
case RUSSIAN_KOI8_R:
case RUSSIAN_CP1251:
case MSFT_CP1252:
case RUSSIAN_KOI8_RU:
case MSFT_CP1250:
case ISO_8859_15:
case MSFT_CP1254:
case MSFT_CP1257:
case ISO_8859_11:
case MSFT_CP874:
case MSFT_CP1256:
case MSFT_CP1255:
case ISO_8859_8_I:
case HEBREW_VISUAL:
case CZECH_CP852:
case MSFT_CP1253:
case RUSSIAN_CP866:
case ISO_8859_13:
case GBK:
case GB18030:
case BIG5_HKSCS:
case MACINTOSH_ROMAN:
return true;
default:
return false;
}
}
// To be an 8-bit encoding means that there are fewer than 256 symbols.
// Each byte determines a new character; there are no multi-byte sequences.
// TODO: This list could maybe be expanded. Other encodings may be 8-bit.
bool Is8BitEncoding(Encoding e) {
switch (e) {
case ASCII_7BIT:
case ISO_8859_1:
case ISO_8859_2:
case ISO_8859_3:
case ISO_8859_4:
case ISO_8859_5:
case ISO_8859_6:
case ISO_8859_7:
case ISO_8859_8:
case ISO_8859_8_I:
case ISO_8859_9:
case ISO_8859_10:
case ISO_8859_11:
case ISO_8859_13:
case ISO_8859_15:
case MSFT_CP1252:
case MSFT_CP1253:
case MSFT_CP1254:
case MSFT_CP1255:
case MSFT_CP1256:
case MSFT_CP1257:
case RUSSIAN_KOI8_R:
case RUSSIAN_KOI8_RU:
case RUSSIAN_CP866:
return true;
default:
return false;
}
}
bool IsCJKEncoding(Encoding e) {
switch (e) {
case JAPANESE_EUC_JP:
case JAPANESE_SHIFT_JIS:
case JAPANESE_JIS:
case CHINESE_BIG5:
case CHINESE_GB:
case CHINESE_EUC_CN:
case KOREAN_EUC_KR:
case CHINESE_EUC_DEC:
case CHINESE_CNS:
case CHINESE_BIG5_CP950:
case JAPANESE_CP932:
case ISO_2022_KR:
case GBK:
case GB18030:
case BIG5_HKSCS:
case ISO_2022_CN:
case HZ_GB_2312:
return true;
default:
return false;
}
}
bool IsHebrewEncoding(Encoding e) {
return (e == ISO_8859_8 ||
e == ISO_8859_8_I ||
e == MSFT_CP1255 ||
e == HEBREW_VISUAL);
}
bool IsRightToLeftEncoding(Encoding enc) {
switch (enc) {
case MSFT_CP1255:
case MSFT_CP1256:
case ARABIC_ENCODING:
case HEBREW_ENCODING:
case ISO_8859_8_I:
case HEBREW_VISUAL:
return true;
default:
return false;
}
}
bool IsLogicalRightToLeftEncoding(Encoding enc) {
return IsRightToLeftEncoding(enc) && !IsVisualRightToLeftEncoding(enc);
}
// Note that despite an RFC to the contrary, ARABIC_ENCODING (ISO-8859-6)
// is NOT visual.
bool IsVisualRightToLeftEncoding(Encoding enc) {
switch (enc) {
case HEBREW_ENCODING:
case HEBREW_VISUAL:
return true;
default:
return false;
}
}
bool IsIso2022Encoding(Encoding enc) {
return (IsIso2022JpOrVariant(enc) ||
enc == ISO_2022_KR ||
enc == ISO_2022_CN);
}
bool IsIso2022JpOrVariant(Encoding enc) {
return (enc == JAPANESE_JIS ||
enc == KDDI_ISO_2022_JP ||
enc == SOFTBANK_ISO_2022_JP);
}
bool IsShiftJisOrVariant(Encoding enc) {
return (enc == JAPANESE_SHIFT_JIS ||
enc == JAPANESE_CP932 ||
enc == KDDI_SHIFT_JIS ||
enc == DOCOMO_SHIFT_JIS ||
enc == SOFTBANK_SHIFT_JIS);
}
bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc) {
return (enc == KDDI_ISO_2022_JP ||
enc == KDDI_SHIFT_JIS ||
enc == DOCOMO_SHIFT_JIS ||
enc == SOFTBANK_SHIFT_JIS ||
enc == SOFTBANK_ISO_2022_JP);
}
// *************************************************************
// ENCODING NAMES
// EncodingName() [Encoding to name]
// MimeEncodingName() [Encoding to name]
// EncodingFromName() [name to Encoding]
// EncodingNameAliasToEncoding() [name to Encoding]
// default_encoding_name()
// invalid_encoding_name()
// *************************************************************
const char * EncodingName(const Encoding enc) {
if ( (enc < 0) || (enc >= kNumEncodings) )
return invalid_encoding_name();
return kEncodingInfoTable[enc].encoding_name_;
}
// TODO: Unify MimeEncodingName and EncodingName, or determine why
// such a unification is not possible.
const char * MimeEncodingName(Encoding enc) {
if ( (enc < 0) || (enc >= kNumEncodings) )
return ""; // TODO: Should this be invalid_encoding_name()?
return kEncodingInfoTable[enc].mime_encoding_name_;
}
bool EncodingFromName(const char* enc_name, Encoding *encoding) {
*encoding = UNKNOWN_ENCODING;
if ( enc_name == NULL ) return false;
for ( int i = 0; i < kNumEncodings; i++ ) {
if (!base::strcasecmp(enc_name, kEncodingInfoTable[i].encoding_name_) ) {
*encoding = static_cast<Encoding>(i);
return true;
}
}
return false;
}
// The encoding_map maps standard and non-standard encoding-names
// (strings) to Encoding enums. It is used only by
// EncodingNameAliasToEncoding. Note that the map uses
// case-insensitive hash and comparison functions.
typedef std::unordered_map<const char *, Encoding,
CStringAlnumCaseHash,
CStringAlnumCaseEqual> EncodingMap;
static const EncodingMap& GetEncodingMap() {
static EncodingMap encoding_map;
if (!encoding_map.empty()) {
// Already initialized
return encoding_map;
}
// Initialize the map with all the "standard" encoding names,
// i.e., the ones returned by EncodingName and MimeEncodingName.
//
// First, add internal encoding names returned by EncodingName().
for (int i = 0; i < NUM_ENCODINGS; ++i) {
Encoding e = static_cast<Encoding>(i);
// Internal encoding names must be unique.
// The internal names are guaranteed to be unique by the CHECK_EQ.
const char *encoding_name = EncodingName(e);
// CHECK_EQ(0, encoding_map.count(encoding_name))
// << "Duplicate found for " << encoding_name;
encoding_map[encoding_name] = e;
}
// Then, add mime encoding names returned by MimeEncodingName().
// We don't override existing entries, to give precedence to entries
// added earlier.
for (int i = 0; i < NUM_ENCODINGS; ++i) {
Encoding e = static_cast<Encoding>(i);
// Note that MimeEncodingName() can return the same mime encoding
// name for different encoding enums like JAPANESE_SHIFT_JIS and
// KDDI_SHIFT_JIS. In that case, the encoding enum first seen
// will be the value for the encoding name in the map.
const char *mime_encoding_name = MimeEncodingName(e);
if (encoding_map.count(mime_encoding_name) == 0) {
encoding_map[mime_encoding_name] = e;
}
}
// Add some non-standard names: alternate spellings, common typos,
// etc. (It does no harm to add names already in the map.) Note
// that although the map is case-insensitive, by convention the
// keys are written here in lower case. For ease of maintenance,
// they are listed in alphabetical order.
encoding_map["5601"] = KOREAN_EUC_KR;
encoding_map["646"] = ASCII_7BIT;
encoding_map["852"] = CZECH_CP852;
encoding_map["866"] = RUSSIAN_CP866;
encoding_map["8859-1"] = ISO_8859_1;
encoding_map["ansi-1251"] = RUSSIAN_CP1251;
encoding_map["ansi_x3.4-1968"] = ASCII_7BIT;
encoding_map["arabic"] = ISO_8859_6;
encoding_map["ascii"] = ISO_8859_1;
encoding_map["ascii-7-bit"] = ASCII_7BIT; // not iana standard
encoding_map["asmo-708"] = ISO_8859_6;
encoding_map["bhaskar"] = BHASKAR;
encoding_map["big5"] = CHINESE_BIG5;
encoding_map["big5-cp950"] = CHINESE_BIG5_CP950; // not iana standard
encoding_map["big5-hkscs"] = BIG5_HKSCS;
encoding_map["chinese"] = CHINESE_GB;
encoding_map["cns"] = CHINESE_CNS; // not iana standard
encoding_map["cns11643"] = CHINESE_CNS;
encoding_map["cp1250"] = MSFT_CP1250; // not iana standard
encoding_map["cp1251"] = RUSSIAN_CP1251; // not iana standard
encoding_map["cp1252"] = MSFT_CP1252; // not iana standard
encoding_map["cp1253"] = MSFT_CP1253; // not iana standard
encoding_map["cp1254"] = MSFT_CP1254; // not iana standard
encoding_map["cp1255"] = MSFT_CP1255;
encoding_map["cp1256"] = MSFT_CP1256;
encoding_map["cp1257"] = MSFT_CP1257; // not iana standard
encoding_map["cp819"] = ISO_8859_1;
encoding_map["cp852"] = CZECH_CP852;
encoding_map["cp866"] = RUSSIAN_CP866;
encoding_map["cp-866"] = RUSSIAN_CP866;
encoding_map["cp874"] = MSFT_CP874;
encoding_map["cp932"] = JAPANESE_CP932; // not iana standard
encoding_map["cp950"] = CHINESE_BIG5_CP950; // not iana standard
encoding_map["csbig5"] = CHINESE_BIG5;
encoding_map["cseucjpkdfmtjapanese"] = JAPANESE_EUC_JP;
encoding_map["cseuckr"] = KOREAN_EUC_KR;
encoding_map["csgb2312"] = CHINESE_GB;
encoding_map["csibm852"] = CZECH_CP852;
encoding_map["csibm866"] = RUSSIAN_CP866;
encoding_map["csiso2022jp"] = JAPANESE_JIS;
encoding_map["csiso2022kr"] = ISO_2022_KR;
encoding_map["csiso58gb231280"] = CHINESE_GB;
encoding_map["csiso88598i"] = ISO_8859_8_I;
encoding_map["csisolatin1"] = ISO_8859_1;
encoding_map["csisolatin2"] = ISO_8859_2;
encoding_map["csisolatin3"] = ISO_8859_3;
encoding_map["csisolatin4"] = ISO_8859_4;
encoding_map["csisolatin5"] = ISO_8859_9;
encoding_map["csisolatin6"] = ISO_8859_10;
encoding_map["csisolatinarabic"] = ISO_8859_6;
encoding_map["csisolatincyrillic"] = ISO_8859_5;
encoding_map["csisolatingreek"] = ISO_8859_7;
encoding_map["csisolatinhebrew"] = ISO_8859_8;
encoding_map["csksc56011987"] = KOREAN_EUC_KR;
encoding_map["csmacintosh"] = MACINTOSH_ROMAN;
encoding_map["csn-369103"] = CZECH_CSN_369103;
encoding_map["csshiftjis"] = JAPANESE_SHIFT_JIS;
encoding_map["csunicode"] = UTF16BE;
encoding_map["csunicode11"] = UTF16BE;
encoding_map["csunicode11utf7"] = UTF7;
encoding_map["csunicodeascii"] = UTF16BE;
encoding_map["csunicodelatin1"] = UTF16BE;
encoding_map["cyrillic"] = ISO_8859_5;
encoding_map["ecma-114"] = ISO_8859_6;
encoding_map["ecma-118"] = ISO_8859_7;
encoding_map["elot_928"] = ISO_8859_7;
encoding_map["euc"] = CHINESE_EUC_DEC; // not iana standard
encoding_map["euc-cn"] = CHINESE_EUC_CN; // not iana standard
encoding_map["euc-dec"] = CHINESE_EUC_DEC; // not iana standard
encoding_map["euc-jp"] = JAPANESE_EUC_JP;
encoding_map["euc-kr"] = KOREAN_EUC_KR;
encoding_map["eucgb2312_cn"] = CHINESE_GB;
encoding_map["gb"] = CHINESE_GB; // not iana standard
encoding_map["gb18030"] = GB18030;
encoding_map["gb2132"] = CHINESE_GB; // common typo
encoding_map["gb2312"] = CHINESE_GB;
encoding_map["gb_2312-80"] = CHINESE_GB;
encoding_map["gbk"] = GBK;
encoding_map["greek"] = ISO_8859_7;
encoding_map["greek8"] = ISO_8859_7;
encoding_map["hebrew"] = ISO_8859_8;
encoding_map["htchanakya"] = HTCHANAKYA;
encoding_map["hz-gb-2312"] = HZ_GB_2312;
encoding_map["ibm819"] = ISO_8859_1;
encoding_map["ibm852"] = CZECH_CP852;
encoding_map["ibm874"] = MSFT_CP874;
encoding_map["iso-10646"] = UTF16BE;
encoding_map["iso-10646-j-1"] = UTF16BE;
encoding_map["iso-10646-ucs-2"] = UNICODE;
encoding_map["iso-10646-ucs-4"] = UTF32BE;
encoding_map["iso-10646-ucs-basic"] = UTF16BE;
encoding_map["iso-10646-unicode-latin1"] = UTF16BE;
encoding_map["iso-2022-cn"] = ISO_2022_CN;
encoding_map["iso-2022-jp"] = JAPANESE_JIS;
encoding_map["iso-2022-kr"] = ISO_2022_KR;
encoding_map["iso-8559-1"] = ISO_8859_1; // common typo
encoding_map["iso-874"] = MSFT_CP874;
encoding_map["iso-8858-1"] = ISO_8859_1; // common typo
// iso-8859-0 was a temporary name, eventually renamed iso-8859-15
encoding_map["iso-8859-0"] = ISO_8859_15;
encoding_map["iso-8859-1"] = ISO_8859_1;
encoding_map["iso-8859-10"] = ISO_8859_10;
encoding_map["iso-8859-11"] = ISO_8859_11;
encoding_map["iso-8859-13"] = ISO_8859_13;
encoding_map["iso-8859-15"] = ISO_8859_15;
encoding_map["iso-8859-2"] = ISO_8859_2;
encoding_map["iso-8859-3"] = ISO_8859_3;
encoding_map["iso-8859-4"] = ISO_8859_4;
encoding_map["iso-8859-5"] = ISO_8859_5;
encoding_map["iso-8859-6"] = ISO_8859_6;
encoding_map["iso-8859-7"] = ISO_8859_7;
encoding_map["iso-8859-8"] = ISO_8859_8;
encoding_map["iso-8859-8-i"] = ISO_8859_8_I;
encoding_map["iso-8859-9"] = ISO_8859_9;
encoding_map["iso-9959-1"] = ISO_8859_1; // common typo
encoding_map["iso-ir-100"] = ISO_8859_1;
encoding_map["iso-ir-101"] = ISO_8859_2;
encoding_map["iso-ir-109"] = ISO_8859_3;
encoding_map["iso-ir-110"] = ISO_8859_4;
encoding_map["iso-ir-126"] = ISO_8859_7;
encoding_map["iso-ir-127"] = ISO_8859_6;
encoding_map["iso-ir-138"] = ISO_8859_8;
encoding_map["iso-ir-144"] = ISO_8859_5;
encoding_map["iso-ir-148"] = ISO_8859_9;
encoding_map["iso-ir-149"] = KOREAN_EUC_KR;
encoding_map["iso-ir-157"] = ISO_8859_10;
encoding_map["iso-ir-58"] = CHINESE_GB;
encoding_map["iso-latin-1"] = ISO_8859_1;
encoding_map["iso_2022-cn"] = ISO_2022_CN;
encoding_map["iso_2022-kr"] = ISO_2022_KR;
encoding_map["iso_8859-1"] = ISO_8859_1;
encoding_map["iso_8859-10:1992"] = ISO_8859_10;
encoding_map["iso_8859-11"] = ISO_8859_11;
encoding_map["iso_8859-13"] = ISO_8859_13;
encoding_map["iso_8859-15"] = ISO_8859_15;
encoding_map["iso_8859-1:1987"] = ISO_8859_1;
encoding_map["iso_8859-2"] = ISO_8859_2;
encoding_map["iso_8859-2:1987"] = ISO_8859_2;
encoding_map["iso_8859-3"] = ISO_8859_3;
encoding_map["iso_8859-3:1988"] = ISO_8859_3;
encoding_map["iso_8859-4"] = ISO_8859_4;
encoding_map["iso_8859-4:1988"] = ISO_8859_4;
encoding_map["iso_8859-5"] = ISO_8859_5;
encoding_map["iso_8859-5:1988"] = ISO_8859_5;
encoding_map["iso_8859-6"] = ISO_8859_6;
encoding_map["iso_8859-6:1987"] = ISO_8859_6;
encoding_map["iso_8859-7"] = ISO_8859_7;
encoding_map["iso_8859-7:1987"] = ISO_8859_7;
encoding_map["iso_8859-8"] = ISO_8859_8;
encoding_map["iso_8859-8:1988:"] = ISO_8859_8;
encoding_map["iso_8859-9"] = ISO_8859_9;
encoding_map["iso_8859-9:1989"] = ISO_8859_9;
encoding_map["jagran"] = JAGRAN;
encoding_map["jis"] = JAPANESE_JIS; // not iana standard
encoding_map["koi8-cs"] = CZECH_CSN_369103;
encoding_map["koi8-r"] = RUSSIAN_KOI8_R;
encoding_map["koi8-ru"] = RUSSIAN_KOI8_RU; // not iana standard
encoding_map["koi8-u"] = RUSSIAN_KOI8_RU;
encoding_map["koi8r"] = RUSSIAN_KOI8_R; // not iana standard
encoding_map["koi8u"] = RUSSIAN_KOI8_RU; // not iana standard
encoding_map["korean"] = KOREAN_EUC_KR; // i assume this is what is meant
encoding_map["ks-c-5601"] = KOREAN_EUC_KR; // not iana standard
encoding_map["ks-c-5601-1987"] = KOREAN_EUC_KR; // not iana standard
encoding_map["ks_c_5601-1989"] = KOREAN_EUC_KR;
encoding_map["ksc"] = KOREAN_EUC_KR; // not iana standard
encoding_map["l1"] = ISO_8859_1;
encoding_map["l2"] = ISO_8859_2;
encoding_map["l3"] = ISO_8859_3;
encoding_map["l4"] = ISO_8859_4;
encoding_map["l5"] = ISO_8859_9;
encoding_map["l6"] = ISO_8859_10;
encoding_map["latin-1"] = ISO_8859_1; // not iana standard
encoding_map["latin1"] = ISO_8859_1;
encoding_map["latin2"] = ISO_8859_2;
encoding_map["latin3"] = ISO_8859_3;
encoding_map["latin4"] = ISO_8859_4;
encoding_map["latin5"] = ISO_8859_9;
encoding_map["latin6"] = ISO_8859_10;
encoding_map["mac"] = MACINTOSH_ROMAN;
encoding_map["macintosh"] = MACINTOSH_ROMAN;
encoding_map["macintosh-roman"] = MACINTOSH_ROMAN;
encoding_map["ms932"] = JAPANESE_CP932; // not iana standard
encoding_map["ms_kanji"] = JAPANESE_CP932;
encoding_map["shift-jis"] = JAPANESE_SHIFT_JIS;
encoding_map["shift_jis"] = JAPANESE_SHIFT_JIS;
encoding_map["sjis"] = JAPANESE_SHIFT_JIS; // not iana standard
encoding_map["sjs"] = JAPANESE_SHIFT_JIS; // not iana standard
encoding_map["sun_eu_greek"] = ISO_8859_7;
encoding_map["tab"] = TAMIL_BI;
encoding_map["tam"] = TAMIL_MONO;
encoding_map["tis-620"] = ISO_8859_11;
encoding_map["tscii"] = TSCII;
encoding_map["un"] = UNKNOWN_ENCODING; // not iana standard
encoding_map["unicode"] = UNICODE; // not iana standard
encoding_map["unicode-1-1-utf-7"] = UTF7;
encoding_map["unicode-1-1-utf-8"] = UTF8;
encoding_map["unicode-2-0-utf-7"] = UTF7;
encoding_map["unknown"] = UNKNOWN_ENCODING; // not iana standard
encoding_map["us"] = ISO_8859_1;
encoding_map["us-ascii"] = ISO_8859_1;
encoding_map["utf-16be"] = UTF16BE;
encoding_map["utf-16le"] = UTF16LE;
encoding_map["utf-32be"] = UTF32BE;
encoding_map["utf-32le"] = UTF32LE;
encoding_map["utf-7"] = UTF7;
encoding_map["utf-8"] = UTF8;
encoding_map["utf7"] = UTF7;
encoding_map["utf8"] = UTF8; // not iana standard
encoding_map["visual"] = HEBREW_VISUAL;
encoding_map["win-1250"] = MSFT_CP1250; // not iana standard
encoding_map["win-1251"] = RUSSIAN_CP1251; // not iana standard
encoding_map["window-874"] = MSFT_CP874;
encoding_map["windows-1250"] = MSFT_CP1250;
encoding_map["windows-1251"] = RUSSIAN_CP1251;
encoding_map["windows-1252"] = MSFT_CP1252;
encoding_map["windows-1253"] = MSFT_CP1253;
encoding_map["windows-1254"] = MSFT_CP1254;
encoding_map["windows-1255"] = MSFT_CP1255;
encoding_map["windows-1256"] = MSFT_CP1256;
encoding_map["windows-1257"] = MSFT_CP1257;
encoding_map["windows-31j"] = JAPANESE_CP932;
encoding_map["windows-874"] = MSFT_CP874;
encoding_map["windows-936"] = GBK;
encoding_map["x-big5"] = CHINESE_BIG5;
encoding_map["x-binaryenc"] = BINARYENC; // not iana standard
encoding_map["x-cp1250"] = MSFT_CP1250;
encoding_map["x-cp1251"] = RUSSIAN_CP1251;
encoding_map["x-cp1252"] = MSFT_CP1252;
encoding_map["x-cp1253"] = MSFT_CP1253;
encoding_map["x-cp1254"] = MSFT_CP1254;
encoding_map["x-cp1255"] = MSFT_CP1255;
encoding_map["x-cp1256"] = MSFT_CP1256;
encoding_map["x-cp1257"] = MSFT_CP1257;
encoding_map["x-euc-jp"] = JAPANESE_EUC_JP;
encoding_map["x-euc-tw"] = CHINESE_CNS;
encoding_map["x-gbk"] = GBK;
encoding_map["x-iso-10646-ucs-2-be"] = UTF16BE;
encoding_map["x-iso-10646-ucs-2-le"] = UTF16LE;
encoding_map["x-iso-10646-ucs-4-be"] = UTF32BE;
encoding_map["x-iso-10646-ucs-4-le"] = UTF32LE;
encoding_map["x-jis"] = JAPANESE_JIS; // not iana standard
encoding_map["x-mac-roman"] = MACINTOSH_ROMAN;
encoding_map["x-shift_jis"] = JAPANESE_SHIFT_JIS; // not iana standard
encoding_map["x-sjis"] = JAPANESE_SHIFT_JIS;
encoding_map["x-unicode-2-0-utf-7"] = UTF7;
encoding_map["x-utf8utf8"] = UTF8UTF8; // not iana standard
encoding_map["x-x-big5"] = CHINESE_BIG5;
encoding_map["zh_cn.euc"] = CHINESE_GB;
encoding_map["zh_tw-big5"] = CHINESE_BIG5;
encoding_map["zh_tw-euc"] = CHINESE_CNS;
// Remove they entry for the empty string, if any.
encoding_map.erase("");
return encoding_map;
}
// ----------------------------------------------------------------------
// EncodingNameAliasToEncoding()
//
// This function takes an encoding name/alias and returns the Encoding
// enum. The input is case insensitive. It is the union of the common
// IANA standard names, the charset names used in Netscape Navigator,
// and some common names we have been using.
// See: http://www.iana.org/assignments/character-sets
// http://physics.hallym.ac.kr/resource/relnotes/windows-2.0.html
//
// UNKNOWN_ENCODING is returned if none matches.
//
// TODO: Check if it is possible to remove the non-standard,
// non-netscape-use names. It is because this routine is used for
// encoding detections from html meta info. Non-standard names may
// introduce noise on encoding detection.
//
// TODO: Unify EncodingNameAliasToEncoding and EncodingFromName,
// or determine why such a unification is not possible.
// ----------------------------------------------------------------------
Encoding EncodingNameAliasToEncoding(const char *encoding_name) {
if (!encoding_name) {
return UNKNOWN_ENCODING;
}
const EncodingMap& encoding_map = GetEncodingMap();
EncodingMap::const_iterator emi = encoding_map.find(encoding_name);
if (emi != encoding_map.end()) {
return emi->second;
} else {
return UNKNOWN_ENCODING;
}
}
const char* default_encoding_name() {
return kEncodingInfoTable[LATIN1].encoding_name_;
}
static const char* const kInvalidEncodingName = "invalid_encoding";
const char *invalid_encoding_name() {
return kInvalidEncodingName;
}
// *************************************************************
// Miscellany
// *************************************************************
Encoding PreferredWebOutputEncoding(Encoding enc) {
return IsValidEncoding(enc)
? kEncodingInfoTable[enc].preferred_web_output_encoding_
: UTF8;
}

299
app/src/main/jni/util/encodings/encodings.h
Unescape View File

@ -0,0 +1,299 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_ENCODINGS_ENCODINGS_H_
#define UTIL_ENCODINGS_ENCODINGS_H_
// This interface defines the Encoding enum and various functions that
// depend only on Encoding values.
// A hash-function for Encoding, hash<Encoding>, is defined in
// i18n/encodings/public/encodings-hash.h
// On some Windows projects, UNICODE may be defined, which would prevent the
// Encoding enum below from compiling. Note that this is a quick fix that does
// not break any existing projects. The UNICODE enum may someday be changed
// to something more specific and non-colliding, but this involves careful
// testing of changes in many other projects.
#undef UNICODE
// NOTE: The Encoding enum must always start at 0. This assumption has
// been made and used.
#ifndef SWIG
#include "util/encodings/encodings.pb.h"
#else
// TODO: Include a SWIG workaround header file.
#endif
const int kNumEncodings = NUM_ENCODINGS;
// some of the popular encoding aliases
// TODO: Make these static const Encoding values instead of macros.
#define LATIN1 ISO_8859_1
#define LATIN2 ISO_8859_2
#define LATIN3 ISO_8859_3
#define LATIN4 ISO_8859_4
#define CYRILLIC ISO_8859_5
#define ARABIC_ENCODING ISO_8859_6 // avoiding the same name as language
#define GREEK_ENCODING ISO_8859_7 // avoiding the same name as language
#define HEBREW_ENCODING ISO_8859_8 // avoiding the same name as language
#define LATIN5 ISO_8859_9
#define LATIN6 ISO_8859_10
#define KOREAN_HANGUL KOREAN_EUC_KR
// The default Encoding (LATIN1).
Encoding default_encoding();
// *************************************************************
// Encoding predicates
// IsValidEncoding()
// IsEncEncCompatible
// IsSupersetOfAscii7Bit
// Is8BitEncoding
// IsCJKEncoding
// IsHebrewEncoding
// IsRightToLeftEncoding
// IsLogicalRightToLeftEncoding
// IsVisualRightToLeftEncoding
// IsIso2022Encoding
// IsIso2022JpOrVariant
// IsShiftJisOrVariant
// IsJapaneseCellPhoneCarrierSpecificEncoding
// *************************************************************
// IsValidEncoding
// ===================================
//
// Function to check if the input language enum is within range.
//
bool IsValidEncoding(Encoding enc);
//
// IsEncEncCompatible
// ------------------
//
// This function is to determine whether or not converting from the
// first encoding to the second requires any changes to the underlying
// text (e.g. ASCII_7BIT is a subset of UTF8).
//
// TODO: the current implementation is likely incomplete. It would be
// good to consider the full matrix of all pairs of encodings and to fish out
// all compatible pairs.
//
bool IsEncEncCompatible(const Encoding from, const Encoding to);
// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
// encoding represent the same characters as they do in ISO_8859_1.
// WARNING: This function does not currently return true for all encodings that
// are supersets of Ascii 7-bit.
bool IsSupersetOfAscii7Bit(Encoding e);
// To be an 8-bit encoding means that there are fewer than 256 symbols.
// Each byte determines a new character; there are no multi-byte sequences.
// WARNING: This function does not currently return true for all encodings that
// are 8-bit encodings.
bool Is8BitEncoding(Encoding e);
// IsCJKEncoding
// -------------
//
// This function returns true if the encoding is either Chinese
// (simplified or traditional), Japanese, or Korean. Note: UTF8 is not
// considered a CJK encoding.
bool IsCJKEncoding(Encoding e);
// IsHebrewEncoding
// -------------
//
// This function returns true if the encoding is a Hebrew specific
// encoding (not UTF8, etc).
bool IsHebrewEncoding(Encoding e);
// IsRightToLeftEncoding
// ---------------------
//
// Returns true if the encoding is a right-to-left encoding.
//
// Note that the name of this function is somewhat misleading. There is nothing
// "right to left" about these encodings. They merely contain code points for
// characters in RTL languages such as Hebrew and Arabic. But this is also
// true for UTF-8.
//
// TODO: Get rid of this function. The only special-case we
// should need to worry about are visual encodings. Anything we
// need to do for all 'RTL' encodings we need to do for UTF-8 as well.
bool IsRightToLeftEncoding(Encoding enc);
// IsLogicalRightToLeftEncoding
// ----------------------------
//
// Returns true if the encoding is a logical right-to-left encoding.
// Logical right-to-left encodings are those that the browser renders
// right-to-left and applies the BiDi algorithm to. Therefore the characters
// appear in reading order in the file, and indexing, snippet generation etc.
// should all just work with no special processing.
//
// TODO: Get rid of this function. The only special-case we
// should need to worry about are visual encodings.
bool IsLogicalRightToLeftEncoding(Encoding enc);
// IsVisualRightToLeftEncoding
// ---------------------------
//
// Returns true if the encoding is a visual right-to-left encoding.
// Visual right-to-left encodings are those that the browser renders
// left-to-right and does not apply the BiDi algorithm to. Therefore each
// line appears in reverse order in the file, lines are manually wrapped
// by abusing <br> or <p> tags, etc. Visual RTL encoding is a relic of
// the prehistoric days when browsers couldn't render right-to-left, but
// unfortunately some visual pages persist to this day. These documents require
// special processing so that we don't index or snippet them with each line
// reversed.
bool IsVisualRightToLeftEncoding(Encoding enc);
// IsIso2022Encoding
// -----------------
//
// Returns true if the encoding is a kind of ISO 2022 such as
// ISO-2022-JP.
bool IsIso2022Encoding(Encoding enc);
// IsIso2022JpOrVariant
// --------------------
//
// Returns true if the encoding is ISO-2022-JP or a variant such as
// KDDI's ISO-2022-JP.
bool IsIso2022JpOrVariant(Encoding enc);
// IsShiftJisOrVariant
// --------------------
//
// Returns true if the encoding is Shift_JIS or a variant such as
// KDDI's Shift_JIS.
bool IsShiftJisOrVariant(Encoding enc);
// IsJapanesCellPhoneCarrierSpecificEncoding
// -----------------------------------------
//
// Returns true if it's Japanese cell phone carrier specific encoding
// such as KDDI_SHIFT_JIS.
bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc);
// *************************************************************
// ENCODING NAMES
//
// This interface defines a standard name for each valid encoding, and
// a standard name for invalid encodings. (Some names use all upper
// case, but others use mixed case.)
//
// EncodingName() [Encoding to name]
// MimeEncodingName() [Encoding to name]
// EncodingFromName() [name to Encoding]
// EncodingNameAliasToEncoding() [name to Encoding]
// default_encoding_name()
// invalid_encoding_name()
// *************************************************************
// EncodingName
// ------------
//
// Given the encoding, returns its standard name.
// Return invalid_encoding_name() if the encoding is invalid.
//
const char* EncodingName(Encoding enc);
//
// MimeEncodingName
// ----------------
//
// Return the "preferred MIME name" of an encoding.
//
// This name is suitable for using in HTTP headers, HTML tags,
// and as the "charset" parameter of a MIME Content-Type.
const char* MimeEncodingName(Encoding enc);
// The maximum length of an encoding name
const int kMaxEncodingNameSize = 50;
// The standard name of the default encoding.
const char* default_encoding_name();
// The name used for an invalid encoding.
const char* invalid_encoding_name();
// EncodingFromName
// ----------------
//
// If enc_name matches the standard name of an Encoding, using a
// case-insensitive comparison, set *encoding to that Encoding and
// return true. Otherwise set *encoding to UNKNOWN_ENCODING and
// return false.
//
// REQUIRES: encoding must not be NULL.
//
bool EncodingFromName(const char* enc_name, Encoding *encoding);
//
// EncodingNameAliasToEncoding
// ---------------------------
//
// If enc_name matches the standard name or an alias of an Encoding,
// using a case-insensitive comparison, return that
// Encoding. Otherwise, return UNKNOWN_ENCODING.
//
// Aliases include most mime-encoding names (e.g., "ISO-8859-7" for
// GREEK), alternate names (e.g., "cyrillic" for ISO_8859_5) and
// common variations with hyphens and underscores (e.g., "koi8-u" and
// "koi8u" for RUSSIAN_KOI8_R).
Encoding EncodingNameAliasToEncoding(const char *enc_name);
// *************************************************************
// Miscellany
// *************************************************************
// PreferredWebOutputEncoding
// --------------------------
//
// Some multi-byte encodings use byte values that coincide with the
// ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
// can misinterpret these, as indicated in an external XSS report from
// 2007-02-15. Here, we map these dangerous encodings to safer ones. We
// also use UTF8 instead of encodings that we don't support in our
// output, and we generally try to be conservative in what we send out.
// Where the client asks for single- or double-byte encodings that are
// not as common, we substitute a more common single- or double-byte
// encoding, if there is one, thereby preserving the client's intent
// to use less space than UTF-8. This also means that characters
// outside the destination set will be converted to HTML NCRs (&#NNN;)
// if requested.
Encoding PreferredWebOutputEncoding(Encoding enc);
#endif // UTIL_ENCODINGS_ENCODINGS_H_

181
app/src/main/jni/util/encodings/encodings.pb.h
Unescape View File

@ -0,0 +1,181 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_ENCODINGS_ENCODINGS_PB_H_
#define UTIL_ENCODINGS_ENCODINGS_PB_H_
enum Encoding {
ISO_8859_1 = 0, // Teragram ASCII
ISO_8859_2 = 1, // Teragram Latin2
ISO_8859_3 = 2, // in BasisTech but not in Teragram
ISO_8859_4 = 3, // Teragram Latin4
ISO_8859_5 = 4, // Teragram ISO-8859-5
ISO_8859_6 = 5, // Teragram Arabic
ISO_8859_7 = 6, // Teragram Greek
ISO_8859_8 = 7, // Teragram Hebrew
ISO_8859_9 = 8, // in BasisTech but not in Teragram
ISO_8859_10 = 9, // in BasisTech but not in Teragram
JAPANESE_EUC_JP = 10, // Teragram EUC_JP
JAPANESE_SHIFT_JIS = 11, // Teragram SJS
JAPANESE_JIS = 12, // Teragram JIS
CHINESE_BIG5 = 13, // Teragram BIG5
CHINESE_GB = 14, // Teragram GB
CHINESE_EUC_CN = 15, // Misnamed. Should be EUC_TW. Was Basis Tech
// CNS11643EUC, before that Teragram EUC-CN(!)
// See //i18n/basistech/basistech_encodings.h
KOREAN_EUC_KR = 16, // Teragram KSC
UNICODE = 17, // Teragram Unicode
CHINESE_EUC_DEC = 18, // Misnamed. Should be EUC_TW. Was Basis Tech
// CNS11643EUC, before that Teragram EUC.
CHINESE_CNS = 19, // Misnamed. Should be EUC_TW. Was Basis Tech
// CNS11643EUC, before that Teragram CNS.
CHINESE_BIG5_CP950 = 20, // Teragram BIG5_CP950
JAPANESE_CP932 = 21, // Teragram CP932
UTF8 = 22,
UNKNOWN_ENCODING = 23,
ASCII_7BIT = 24, // ISO_8859_1 with all characters <= 127.
// Should be present only in the crawler
// and in the repository,
// *never* as a result of Document::encoding().
RUSSIAN_KOI8_R = 25, // Teragram KOI8R
RUSSIAN_CP1251 = 26, // Teragram CP1251
//----------------------------------------------------------
// These are _not_ output from teragram. Instead, they are as
// detected in the headers of usenet articles.
MSFT_CP1252 = 27, // 27: CP1252 aka MSFT euro ascii
RUSSIAN_KOI8_RU = 28, // CP21866 aka KOI8-U, used for Ukrainian.
// Misnamed, this is _not_ KOI8-RU but KOI8-U.
// KOI8-U is used much more often than KOI8-RU.
MSFT_CP1250 = 29, // CP1250 aka MSFT eastern european
ISO_8859_15 = 30, // aka ISO_8859_0 aka ISO_8859_1 euroized
//----------------------------------------------------------
//----------------------------------------------------------
// These are in BasisTech but not in Teragram. They are
// needed for new interface languages. Now detected by
// research langid
MSFT_CP1254 = 31, // used for Turkish
MSFT_CP1257 = 32, // used in Baltic countries
//----------------------------------------------------------
//----------------------------------------------------------
//----------------------------------------------------------
// New encodings detected by Teragram
ISO_8859_11 = 33, // aka TIS-620, used for Thai
MSFT_CP874 = 34, // used for Thai
MSFT_CP1256 = 35, // used for Arabic
//----------------------------------------------------------
// Detected as ISO_8859_8 by Teragram, but can be found in META tags
MSFT_CP1255 = 36, // Logical Hebrew Microsoft
ISO_8859_8_I = 37, // Iso Hebrew Logical
HEBREW_VISUAL = 38, // Iso Hebrew Visual
//----------------------------------------------------------
//----------------------------------------------------------
// Detected by research langid
CZECH_CP852 = 39,
CZECH_CSN_369103 = 40, // aka ISO_IR_139 aka KOI8_CS
MSFT_CP1253 = 41, // used for Greek
RUSSIAN_CP866 = 42,
//----------------------------------------------------------
//----------------------------------------------------------
// Handled by iconv in glibc
ISO_8859_13 = 43,
ISO_2022_KR = 44,
GBK = 45,
GB18030 = 46,
BIG5_HKSCS = 47,
ISO_2022_CN = 48,
//-----------------------------------------------------------
// Detected by xin liu's detector
// Handled by transcoder
// (Indic encodings)
TSCII = 49,
TAMIL_MONO = 50,
TAMIL_BI = 51,
JAGRAN = 52,
MACINTOSH_ROMAN = 53,
UTF7 = 54,
BHASKAR = 55, // Indic encoding - Devanagari
HTCHANAKYA = 56, // 56 Indic encoding - Devanagari
//-----------------------------------------------------------
// These allow a single place (inputconverter and outputconverter)
// to do UTF-16 <==> UTF-8 bulk conversions and UTF-32 <==> UTF-8
// bulk conversions, with interchange-valid checking on input and
// fallback if needed on ouput.
UTF16BE = 57, // big-endian UTF-16
UTF16LE = 58, // little-endian UTF-16
UTF32BE = 59, // big-endian UTF-32
UTF32LE = 60, // little-endian UTF-32
//-----------------------------------------------------------
//-----------------------------------------------------------
// An encoding that means "This is not text, but it may have some
// simple ASCII text embedded". Intended input conversion (not yet
// implemented) is to keep strings of >=4 seven-bit ASCII characters
// (follow each kept string with an ASCII space), delete the rest of
// the bytes. This will pick up and allow indexing of e.g. captions
// in JPEGs. No output conversion needed.
BINARYENC = 61,
//-----------------------------------------------------------
//-----------------------------------------------------------
// Some Web pages allow a mixture of HZ-GB and GB-2312 by using
// ~{ ... ~} for 2-byte pairs, and the browsers support this.
HZ_GB_2312 = 62,
//-----------------------------------------------------------
//-----------------------------------------------------------
// Some external vendors make the common input error of
// converting MSFT_CP1252 to UTF8 *twice*. No output conversion needed.
UTF8UTF8 = 63,
//-----------------------------------------------------------
//-----------------------------------------------------------
// Handled by transcoder for tamil language specific font
// encodings without the support for detection at present.
TAM_ELANGO = 64, // Elango - Tamil
TAM_LTTMBARANI = 65, // Barani - Tamil
TAM_SHREE = 66, // Shree - Tamil
TAM_TBOOMIS = 67, // TBoomis - Tamil
TAM_TMNEWS = 68, // TMNews - Tamil
TAM_WEBTAMIL = 69, // Webtamil - Tamil
//-----------------------------------------------------------
//-----------------------------------------------------------
// Shift_JIS variants used by Japanese cell phone carriers.
KDDI_SHIFT_JIS = 70,
DOCOMO_SHIFT_JIS = 71,
SOFTBANK_SHIFT_JIS = 72,
// ISO-2022-JP variants used by KDDI and SoftBank.
KDDI_ISO_2022_JP = 73,
SOFTBANK_ISO_2022_JP = 74,
//-----------------------------------------------------------
NUM_ENCODINGS = 75, // Always keep this at the end. It is not a
// valid Encoding enum, it is only used to
// indicate the total number of Encodings.
};
#endif // UTIL_ENCODINGS_ENCODINGS_PB_H_

34
app/src/main/jni/util/encodings/encodings_unittest.cc
Unescape View File

@ -0,0 +1,34 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#include "util/encodings/encodings.h"
#include "gtest/gtest.h"
TEST(EncodingsTest, EncodingNameAliasToEncoding) {
// Test that cases, non-alpha-numeric chars are ignored.
EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso_8859_1"));
EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso-8859-1"));
// Test that spaces are ignored.
EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF8"));
EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF 8"));
EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF-8"));
// Test alphanumeric differences are counted.
EXPECT_NE(UTF8, EncodingNameAliasToEncoding("UTF-7"));
EXPECT_NE(KOREAN_EUC_KR, EncodingNameAliasToEncoding("euc-jp"));
}

349
app/src/main/jni/util/languages/languages.cc
Unescape View File

@ -0,0 +1,349 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#include "util/languages/languages.h"
#include "util/basictypes.h"
#include "util/string_util.h"
Language default_language() {return ENGLISH;}
// Language names and codes
struct LanguageInfo {
const char * language_name_;
const char * language_code_639_1_; // the ISO-639-1 code for the language
const char * language_code_639_2_; // the ISO-639-2 code for the language
const char * language_code_other_; // some nonstandard code for the language
};
static const LanguageInfo kLanguageInfoTable[] = {
{ "ENGLISH", "en", "eng", NULL},
{ "DANISH", "da", "dan", NULL},
{ "DUTCH", "nl", "dut", NULL},
{ "FINNISH", "fi", "fin", NULL},
{ "FRENCH", "fr", "fre", NULL},
{ "GERMAN", "de", "ger", NULL},
{ "HEBREW", "he", "heb", NULL},
{ "ITALIAN", "it", "ita", NULL},
{ "Japanese", "ja", "jpn", NULL},
{ "Korean", "ko", "kor", NULL},
{ "NORWEGIAN", "nb", "nor", NULL},
{ "POLISH", "pl", "pol", NULL},
{ "PORTUGUESE", "pt", "por", NULL},
{ "RUSSIAN", "ru", "rus", NULL},
{ "SPANISH", "es", "spa", NULL},
{ "SWEDISH", "sv", "swe", NULL},
{ "Chinese", "zh", "chi", "zh-CN"},
{ "CZECH", "cs", "cze", NULL},
{ "GREEK", "el", "gre", NULL},
{ "ICELANDIC", "is", "ice", NULL},
{ "LATVIAN", "lv", "lav", NULL},
{ "LITHUANIAN", "lt", "lit", NULL},
{ "ROMANIAN", "ro", "rum", NULL},
{ "HUNGARIAN", "hu", "hun", NULL},
{ "ESTONIAN", "et", "est", NULL},
// TODO: Although Teragram has two output names "TG_UNKNOWN_LANGUAGE"
// and "Unknown", they are essentially the same. Need to unify them.
// "un" and "ut" are invented by us, not from ISO-639.
//
{ "TG_UNKNOWN_LANGUAGE", NULL, NULL, "ut"},
{ "Unknown", NULL, NULL, "un"},
{ "BULGARIAN", "bg", "bul", NULL},
{ "CROATIAN", "hr", "scr", NULL},
{ "SERBIAN", "sr", "scc", NULL},
{ "IRISH", "ga", "gle", NULL},
{ "GALICIAN", "gl", "glg", NULL},
// Impossible to tell Tagalog from Filipino at the moment.
// Use ISO 639-2 code for Filipino here.
{ "TAGALOG", NULL, "fil", NULL},
{ "TURKISH", "tr", "tur", NULL},
{ "UKRAINIAN", "uk", "ukr", NULL},
{ "HINDI", "hi", "hin", NULL},
{ "MACEDONIAN", "mk", "mac", NULL},
{ "BENGALI", "bn", "ben", NULL},
{ "INDONESIAN", "id", "ind", NULL},
{ "LATIN", "la", "lat", NULL},
{ "MALAY", "ms", "may", NULL},
{ "MALAYALAM", "ml", "mal", NULL},
{ "WELSH", "cy", "wel", NULL},
{ "NEPALI", "ne", "nep", NULL},
{ "TELUGU", "te", "tel", NULL},
{ "ALBANIAN", "sq", "alb", NULL},
{ "TAMIL", "ta", "tam", NULL},
{ "BELARUSIAN", "be", "bel", NULL},
{ "JAVANESE", "jw", "jav", NULL},
{ "OCCITAN", "oc", "oci", NULL},
{ "URDU", "ur", "urd", NULL},
{ "BIHARI", "bh", "bih", NULL},
{ "GUJARATI", "gu", "guj", NULL},
{ "THAI", "th", "tha", NULL},
{ "ARABIC", "ar", "ara", NULL},
{ "CATALAN", "ca", "cat", NULL},
{ "ESPERANTO", "eo", "epo", NULL},
{ "BASQUE", "eu", "baq", NULL},
{ "INTERLINGUA", "ia", "ina", NULL},
{ "KANNADA", "kn", "kan", NULL},
{ "PUNJABI", "pa", "pan", NULL},
{ "SCOTS_GAELIC", "gd", "gla", NULL},
{ "SWAHILI", "sw", "swa", NULL},
{ "SLOVENIAN", "sl", "slv", NULL},
{ "MARATHI", "mr", "mar", NULL},
{ "MALTESE", "mt", "mlt", NULL},
{ "VIETNAMESE", "vi", "vie", NULL},
{ "FRISIAN", "fy", "fry", NULL},
{ "SLOVAK", "sk", "slo", NULL},
{ "ChineseT",
NULL, NULL, // We intentionally set these 2 fields to NULL to avoid
// confusion between CHINESE_T and CHINESE.
"zh-TW"},
{ "FAROESE", "fo", "fao", NULL},
{ "SUNDANESE", "su", "sun", NULL},
{ "UZBEK", "uz", "uzb", NULL},
{ "AMHARIC", "am", "amh", NULL},
{ "AZERBAIJANI", "az", "aze", NULL},
{ "GEORGIAN", "ka", "geo", NULL},
{ "TIGRINYA", "ti", "tir", NULL},
{ "PERSIAN", "fa", "per", NULL},
{ "BOSNIAN", "bs", "bos", NULL},
{ "SINHALESE", "si", "sin", NULL},
{ "NORWEGIAN_N", "nn", "nno", NULL},
{ "PORTUGUESE_P", NULL, NULL, "pt-PT"},
{ "PORTUGUESE_B", NULL, NULL, "pt-BR"},
{ "XHOSA", "xh", "xho", NULL},
{ "ZULU", "zu", "zul", NULL},
{ "GUARANI", "gn", "grn", NULL},
{ "SESOTHO", "st", "sot", NULL},
{ "TURKMEN", "tk", "tuk", NULL},
{ "KYRGYZ", "ky", "kir", NULL},
{ "BRETON", "br", "bre", NULL},
{ "TWI", "tw", "twi", NULL},
{ "YIDDISH", "yi", "yid", NULL},
{ "SERBO_CROATIAN", "sh", NULL, NULL},
{ "SOMALI", "so", "som", NULL},
{ "UIGHUR", "ug", "uig", NULL},
{ "KURDISH", "ku", "kur", NULL},
{ "MONGOLIAN", "mn", "mon", NULL},
{ "ARMENIAN", "hy", "arm", NULL},
{ "LAOTHIAN", "lo", "lao", NULL},
{ "SINDHI", "sd", "snd", NULL},
{ "RHAETO_ROMANCE", "rm", "roh", NULL},
{ "AFRIKAANS", "af", "afr", NULL},
{ "LUXEMBOURGISH", "lb", "ltz", NULL},
{ "BURMESE", "my", "bur", NULL},
// KHMER is known as Cambodian for Google user interfaces.
{ "KHMER", "km", "khm", NULL},
{ "TIBETAN", "bo", "tib", NULL},
{ "DHIVEHI", "dv", "div", NULL},
{ "CHEROKEE", NULL, "chr", NULL},
{ "SYRIAC", NULL, "syr", NULL},
{ "LIMBU", NULL, NULL, "sit-NP"},
{ "ORIYA", "or", "ori", NULL},
{ "ASSAMESE", "as", "asm", NULL},
{ "CORSICAN", "co", "cos", NULL},
{ "INTERLINGUE", "ie", "ine", NULL},
{ "KAZAKH", "kk", "kaz", NULL},
{ "LINGALA", "ln", "lin", NULL},
{ "MOLDAVIAN", "mo", "mol", NULL},
{ "PASHTO", "ps", "pus", NULL},
{ "QUECHUA", "qu", "que", NULL},
{ "SHONA", "sn", "sna", NULL},
{ "TAJIK", "tg", "tgk", NULL},
{ "TATAR", "tt", "tat", NULL},
{ "TONGA", "to", "tog", NULL},
{ "YORUBA", "yo", "yor", NULL},
{ "CREOLES_AND_PIDGINS_ENGLISH_BASED", NULL, "cpe", NULL},
{ "CREOLES_AND_PIDGINS_FRENCH_BASED", NULL, "cpf", NULL},
{ "CREOLES_AND_PIDGINS_PORTUGUESE_BASED", NULL, "cpp", NULL},
{ "CREOLES_AND_PIDGINS_OTHER", NULL, "crp", NULL},
{ "MAORI", "mi", "mao", NULL},
{ "WOLOF", "wo", "wol", NULL},
{ "ABKHAZIAN", "ab", "abk", NULL},
{ "AFAR", "aa", "aar", NULL},
{ "AYMARA", "ay", "aym", NULL},
{ "BASHKIR", "ba", "bak", NULL},
{ "BISLAMA", "bi", "bis", NULL},
{ "DZONGKHA", "dz", "dzo", NULL},
{ "FIJIAN", "fj", "fij", NULL},
{ "GREENLANDIC", "kl", "kal", NULL},
{ "HAUSA", "ha", "hau", NULL},
{ "HAITIAN_CREOLE", "ht", NULL, NULL},
{ "INUPIAK", "ik", "ipk", NULL},
{ "INUKTITUT", "iu", "iku", NULL},
{ "KASHMIRI", "ks", "kas", NULL},
{ "KINYARWANDA", "rw", "kin", NULL},
{ "MALAGASY", "mg", "mlg", NULL},
{ "NAURU", "na", "nau", NULL},
{ "OROMO", "om", "orm", NULL},
{ "RUNDI", "rn", "run", NULL},
{ "SAMOAN", "sm", "smo", NULL},
{ "SANGO", "sg", "sag", NULL},
{ "SANSKRIT", "sa", "san", NULL},
{ "SISWANT", "ss", "ssw", NULL},
{ "TSONGA", "ts", "tso", NULL},
{ "TSWANA", "tn", "tsn", NULL},
{ "VOLAPUK", "vo", "vol", NULL},
{ "ZHUANG", "za", "zha", NULL},
{ "KHASI", NULL, "kha", NULL},
{ "SCOTS", NULL, "sco", NULL},
{ "GANDA", "lg", "lug", NULL},
{ "MANX", "gv", "glv", NULL},
{ "MONTENEGRIN", NULL, NULL, "sr-ME"},
{ "XX", NULL, NULL, "XX"},
};
COMPILE_ASSERT(arraysize(kLanguageInfoTable) == NUM_LANGUAGES + 1,
kLanguageInfoTable_has_incorrect_length);
// LANGUAGE NAMES
const char* default_language_name() {
return kLanguageInfoTable[ENGLISH].language_name_;
}
static const char* const kInvalidLanguageName = "invalid_language";
const char *invalid_language_name() {
return kInvalidLanguageName;
}
const char* LanguageName(Language lang) {
return IsValidLanguage(lang)
? kLanguageInfoTable[lang].language_name_
: kInvalidLanguageName;
}
// LANGUAGE CODES
// The space before invalid_language_code is intentional. It is used
// to prevent it matching any two letter language code.
//
static const char* const kInvalidLanguageCode = " invalid_language_code";
const char *invalid_language_code() {
return kInvalidLanguageCode;
}
const char * LanguageCode(Language lang) {
if (! IsValidLanguage(lang))
return kInvalidLanguageCode;
const LanguageInfo& info = kLanguageInfoTable[lang];
if (info.language_code_639_1_) {
return info.language_code_639_1_;
} else if (info.language_code_639_2_) {
return info.language_code_639_2_;
} else if (info.language_code_other_) {
return info.language_code_other_;
} else {
return kInvalidLanguageCode;
}
}
const char* default_language_code() {
return kLanguageInfoTable[ENGLISH].language_code_639_1_;
}
const char* LanguageCodeISO639_1(Language lang) {
if (! IsValidLanguage(lang))
return kInvalidLanguageCode;
if (const char* code = kLanguageInfoTable[lang].language_code_639_1_)
return code;
return kInvalidLanguageCode;
}
const char* LanguageCodeISO639_2(Language lang) {
if (! IsValidLanguage(lang))
return kInvalidLanguageCode;
if (const char* code = kLanguageInfoTable[lang].language_code_639_2_)
return code;
return kInvalidLanguageCode;
}
const char* LanguageCodeWithDialects(Language lang) {
if (lang == CHINESE)
return "zh-CN";
return LanguageCode(lang);
}
bool LanguageFromCode(const char* lang_code, Language *language) {
*language = UNKNOWN_LANGUAGE;
if ( lang_code == NULL ) return false;
for ( int i = 0 ; i < kNumLanguages ; i++ ) {
const LanguageInfo& info = kLanguageInfoTable[i];
if ((info.language_code_639_1_ &&
!base::strcasecmp(lang_code, info.language_code_639_1_)) ||
(info.language_code_639_2_ &&
!base::strcasecmp(lang_code, info.language_code_639_2_)) ||
(info.language_code_other_ &&
!base::strcasecmp(lang_code, info.language_code_other_))) {
*language = static_cast<Language>(i);
return true;
}
}
// For convenience, this function can also parse the non-standard
// five-letter language codes "zh-cn" and "zh-tw" which are used by
// front-ends such as GWS to distinguish Simplified from Traditional
// Chinese.
if (!base::strcasecmp(lang_code, "zh-cn") ||
!base::strcasecmp(lang_code, "zh_cn")) {
*language = CHINESE;
return true;
}
if (!base::strcasecmp(lang_code, "zh-tw") ||
!base::strcasecmp(lang_code, "zh_tw")) {
*language = CHINESE_T;
return true;
}
if (!base::strcasecmp(lang_code, "sr-me") ||
!base::strcasecmp(lang_code, "sr_me")) {
*language = MONTENEGRIN;
return true;
}
// Process language-code synonyms.
if (!base::strcasecmp(lang_code, "he")) {
*language = HEBREW; // Use "iw".
return true;
}
if (!base::strcasecmp(lang_code, "in")) {
*language = INDONESIAN; // Use "id".
return true;
}
if (!base::strcasecmp(lang_code, "ji")) {
*language = YIDDISH; // Use "yi".
return true;
}
// Process language-detection synonyms.
// These distinct languages cannot be differentiated by our current
// language-detection algorithms.
if (!base::strcasecmp(lang_code, "fil")) {
*language = TAGALOG;
return true;
}
return false;
}

381
app/src/main/jni/util/languages/languages.h
Unescape View File

@ -0,0 +1,381 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_LANGUAGES_LANGUAGES_H_
#define UTIL_LANGUAGES_LANGUAGES_H_
// This interface defines the Language enum and functions that depend
// only on Language values.
// A hash-function for Language, hash<Language>, is defined in
// i18n/languages/public/languages-hash.h
#ifndef SWIG
// Language enum defined in languages.proto
// Also description on how to add languages.
#include "util/languages/languages.pb.h"
#else
// TODO: Include a header containing swig-compatible enum.
#endif
const int kNumLanguages = NUM_LANGUAGES;
// Return the default language (ENGLISH).
Language default_language();
// *******************************************
// Language predicates
// IsValidLanguage()
// IS_LANGUAGE_UNKNOWN()
// IsCJKLanguage()
// IsChineseLanguage()
// IsNorwegianLanguage()
// IsPortugueseLanguage()
// IsRightToLeftLanguage()
// IsMaybeRightToLeftLanguage()
// IsSameLanguage()
// IsScriptRequiringLongerSnippets()
// *******************************************
// IsValidLanguage
// ===============
//
// Function to check if the input is within range of the Language enum. If
// IsValidLanguage(lang) returns true, it is safe to call
// static_cast<Language>(lang).
//
inline bool IsValidLanguage(int lang) {
return ((lang >= 0) && (lang < kNumLanguages));
}
// Return true if the language is "unknown". (This function was
// previously a macro, hence the spelling in all caps.)
//
inline bool IS_LANGUAGE_UNKNOWN(Language lang) {
return lang == TG_UNKNOWN_LANGUAGE || lang == UNKNOWN_LANGUAGE;
}
// IsCJKLanguage
// -------------
//
// This function returns true if the language is either Chinese
// (simplified or traditional), Japanese, or Korean.
bool IsCJKLanguage(Language lang);
// IsChineseLanguage
// -----------------
//
// This function returns true if the language is either Chinese
// (simplified or traditional)
bool IsChineseLanguage(Language lang);
// IsNorwegianLanguage
// --------------------
//
// This function returns true if the language is any of the Norwegian
// (regular or Nynorsk).
bool IsNorwegianLanguage(Language lang);
// IsPortugueseLanguage
// --------------------
//
// This function returns true if the language is any of the Portuguese
// languages (regular, Portugal or Brazil)
bool IsPortugueseLanguage(Language lang);
// IsSameLanguage
// --------------
//
// WARNING: This function provides only a simple test on the values of
// the two Language arguments. It returns false if either language is
// invalid. It returns true if the language arguments are equal, or
// if they are both Chinese languages, both Norwegian languages, or
// both Portuguese languages, as defined by IsChineseLanguage,
// IsNorwegianLanguage, and IsPortugueseLanguage. Otherwise it returns
// false.
bool IsSameLanguage(Language lang1, Language lang2);
// IsRightToLeftLanguage
// ---------------------
//
// This function returns true if the language is only written right-to-left
// (E.g., Hebrew, Arabic, Persian etc.)
//
// IMPORTANT NOTE: Technically we're talking about scripts, not languages.
// There are languages that can be written in more than one script.
// Examples:
// - Kurdish and Azeri ('AZERBAIJANI') can be written left-to-right in
// Latin or Cyrillic script, and right-to-left in Arabic script.
// - Sindhi and Punjabi are written in different scripts, depending on
// region and dialect.
// - Turkmen used an Arabic script historically, but not any more.
// - Pashto and Uyghur can use Arabic script, but use a Roman script
// on the Internet.
// - Kashmiri and Urdu are written either with Arabic or Devanagari script.
//
// This function only returns true for languages that are always, unequivocally
// written in right-to-left script.
//
// TODO: If we want to do anything special with multi-script languages
// we should create new 'languages' for each language+script, as we do for
// traditional vs. simplified Chinese. However most such languages are rare in
// use and even rarer on the web, so this is unlikely to be something we'll
// be concerned with for a while.
bool IsRightToLeftLanguage(Language lang);
// IsMaybeRightToLeftLanguage
// --------------------------
//
// This function returns true if the language may appear on the web in a
// right-to-left script (E.g., Hebrew, Arabic, Persian, Urdu, Kurdish, etc.)
//
// NOTE: See important notes under IsRightToLeftLanguage(...).
//
// This function returns true for languages that *may* appear on the web in a
// right-to-left script, even if they may also appear in a left-to-right
// script.
//
// This function should typically be used in cases where doing some work on
// left-to-right text would be OK (usually a no-op), and this function is used
// just to cut down on unnecessary work on regular, LTR text.
bool IsMaybeRightToLeftLanguage(Language lang);
// IsScriptRequiringLongerSnippets
// --------------------
//
// This function returns true if the script chracteristics require longer
// snippet length (Devanagari, Bengali, Gurmukhi,
// Gujarati, Oriya, Tamil, Telugu, Kannada, Malayalam).
// COMMENTED OUT TO REDUCE DEPENDENCIES ON GOOGLE3 CODE
// bool IsScriptRequiringLongerSnippets(UnicodeScript script);
// *******************************************
// LANGUAGE NAMES
//
// This interface defines a standard name for each valid Language,
// and a standard name for invalid languages. Some language names use all
// uppercase letters, but others use mixed case.
// LanguageName() [Language to name]
// LanguageEnumName() [language to enum name]
// LanguageFromName() [name to Language]
// default_language_name()
// invalid_language_name()
// *******************************************
// Given a Language, returns its standard name.
// Return invalid_language_name() if the language is invalid.
const char* LanguageName(Language lang);
// Given a Language, return the name of the enum constant for that
// language. In all but a few cases, this is the same as its standard
// name. For example, LanguageName(CHINESE) returns "Chinese", but
// LanguageEnumName(CHINESE) returns "CHINESE". This is intended for
// code that is generating C++ code, where the enum constant is more
// useful than its integer value. Return "NUM_LANGUAGES" if
// the language is invalid.
const char* LanguageEnumName(Language lang);
// The maximum length of a standard language name.
const int kMaxLanguageNameSize = 50;
// The standard name for the default language.
const char* default_language_name();
// The standard name for all invalid languages.
const char* invalid_language_name();
// If lang_name matches the standard name of a Language, using a
// case-insensitive comparison, set *language to that Language and
// return true.
// Otherwise, set *language to UNKNOWN_LANGUAGE and return false.
//
// For backwards compatibility, "HATIAN_CREOLE" is allowed as a name
// for HAITIAN_CREOLE, and "QUECHAU" is allowed as a name for QUECHUA.
// For compatibility with LanguageEnumName, "UNKNOWN_LANGUAGE" is allowed
// as a name for UNKNOWN_LANGUAGE (the return value is true in this case,
// as it is for "Unknown"), and "CHINESE_T" is allowed as a name for
// CHINESE_T (i.e., a synonym for "ChineseT").
//
// REQUIRES: language must not be NULL.
//
bool LanguageFromName(const char* lang_name, Language *language);
// *******************************************
// LANGUAGE CODES
//
// This interface defines a standard code for each valid language, and
// a standard code for invalid languages. These are derived from ISO codes,
// with some Google additions.
// LanguageCode()
// default_language_code()
// invalid_language_code()
// LanguageCodeWithDialects()
// LanguageCodeISO639_1()
// LanguageCodeISO639_2()
// *******************************************
// Given a Language, return its standard code. There are Google-specific codes:
// For CHINESE_T, return "zh-TW".
// For TG_UNKNOWN_LANGUAGE, return "ut".
// For UNKNOWN_LANGUAGE, return "un".
// For PORTUGUESE_P, return "pt-PT".
// For PORTUGUESE_B, return "pt-BR".
// For LIMBU, return "sit-NP".
// For CHEROKEE, return "chr".
// For SYRIAC, return "syr".
// Otherwise return the ISO 639-1 two-letter language code for lang.
// If lang is invalid, return invalid_language_code().
//
// NOTE: See the note below about the codes for Chinese languages.
//
const char* LanguageCode(Language lang);
// The maximum length of a language code.
const int kMaxLanguageCodeSize = 50;
// The standard code for the default language.
const char* default_language_code();
// The standard code for all invalid languages.
const char* invalid_language_code();
// --------------------------------------------
// NOTE: CHINESE LANGUAGE CODES
//
// There are three functions that return codes for Chinese languages.
// LanguageCode(lang) and LanguageCodeWithDialects(lang) are defined here.
// LanguageCode(lang, encoding) is defined in i18n/encodings.lang_enc.h.
// The following list shows the different results.
//
// LanguageCode(CHINESE) returns "zh"
// LanguageCode(CHINESE_T) returns "zh-TW".
//
// LanguageCodeWithDialects(CHINESE) returns "zh-CN".
// LanguageCodeWithDialects(CHINESE_T) returns "zh-TW".
//
// LanguageCode(CHINESE_T, <any encoding>) returns "zh-TW".
// LanguageCode(CHINESE, CHINESE_BIG5) returns "zh-TW".
// LanguageCode(CHINESE, <any other encoding>) returns "zh-CN".
//
// --------------------------------------------
// LanguageCodeWithDialects
// ------------------------
//
// If lang is CHINESE, return "zh-CN". Otherwise return LanguageCode(lang).
const char* LanguageCodeWithDialects(Language lang);
// LanguageCodeISO639_1
// --------------------
//
// Return the ISO 639-1 two-letter language code for lang.
// Return invalid_language_code() if lang is invalid or does not have
// an ISO 639-1 two-letter language code.
const char* LanguageCodeISO639_1(Language lang);
// LanguageCodeISO639_2
// --------------------
//
// Return the ISO 639-2 three-letter language for lang.
// Return invalid_language_code() if lang is invalid or does not have
// an ISO 639-2 three-letter language code.
const char* LanguageCodeISO639_2(Language lang);
// LanguageFromCode
// ----------------
//
// If lang_code matches the code for a Language, using a case-insensitive
// comparison, set *lang to that Language and return true.
// Otherwise, set *lang to UNKNOWN_LANGUAGE and return false.
//
// lang_code can be an ISO 639-1 (two-letter) code, an ISO 639-2
// (three-letter) code, or a Google-specific code (see LanguageCode).
//
// Certain language-code aliases are also allowed:
// For "zh-cn" and "zh_cn", set *lang to CHINESE.
// For "zh-tw" and "zh_tw", set *lang to CHINESE_T.
// For "he", set *lang to HEBREW.
// For "in", set *lang to INDONESIAN.
// For "ji", set *lang to YIDDISH.
// For "fil", set *lang to TAGALOG.
//
// REQUIRES: 'lang' must not be NULL.
bool LanguageFromCode(const char* lang_code, Language *language);
// LanguageFromCodeOrName
// ----------------------
//
// If lang_code_or_name is a language code or a language name.
// set *language to the corresponding Language and return true.
// Otherwise set *language to UNKNOWN_LANGUAGE and return false.
//
bool LanguageFromCodeOrName(const char* lang_code_or_name,
Language* language);
// LanguageNameFromCode
// --------------------
//
// If language_code is the code for a Language (see LanguageFromCode),
// return the standard name of that language (see LanguageName).
// Otherwise return invalid_language_name().
//
const char* LanguageNameFromCode(const char* language_code);
// Miscellany
// LanguageCodeToUnderscoreForm
// ----------------------------
//
// Given a language code, convert the dash "-" to underscore "_".
//
// Specifically, if result_length <= strlen(lang_code), set result[0]
// to '\0' and return false. Otherwise, copy lang_code to result,
// converting every dash to an underscore, converting every character
// before the first dash or underscore to lower case, and converting
// every character after the first dash or underscore to upper
// case. If there is no dash or underscore, convert the entire string
// to lower case.
//
// REQUIRES: 'lang_code' must not be NULL. 'result' must not be NULL.
bool LanguageCodeToUnderscoreForm(const char* lang_code,
char* result,
int result_length);
//
// AlwaysPutInExpectedRestrict
// ---------------------------
//
// For Web pages in certain top-level domains, Web Search always
// applies a "country restrict". If 'tld' matches one of those, using
// a case-SENSITIVE comparison, set *expected_language to the Language
// most commonly found in that top-level domain and return true.
// Otherwise, set *expected_language to UNKNOWN_LANGUAGE and return false.
bool AlwaysPutInExpectedRestrict(const char *tld, Language *expected_language);
#endif // UTIL_LANGUAGES_LANGUAGES_H_

191
app/src/main/jni/util/languages/languages.pb.h
Unescape View File

@ -0,0 +1,191 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_LANGUAGES_LANGUAGES_PB_H_
#define UTIL_LANGUAGES_LANGUAGES_PB_H_
enum Language {
ENGLISH = 0,
DANISH = 1,
DUTCH = 2,
FINNISH = 3,
FRENCH = 4,
GERMAN = 5,
HEBREW = 6,
ITALIAN = 7,
JAPANESE = 8,
KOREAN = 9,
NORWEGIAN = 10,
POLISH = 11,
PORTUGUESE = 12,
RUSSIAN = 13,
SPANISH = 14,
SWEDISH = 15,
CHINESE = 16,
CZECH = 17,
GREEK = 18,
ICELANDIC = 19,
LATVIAN = 20,
LITHUANIAN = 21,
ROMANIAN = 22,
HUNGARIAN = 23,
ESTONIAN = 24,
TG_UNKNOWN_LANGUAGE = 25,
UNKNOWN_LANGUAGE = 26,
BULGARIAN = 27,
CROATIAN = 28,
SERBIAN = 29,
IRISH = 30, // UI only.
GALICIAN = 31,
TAGALOG = 32, // Tagalog (tl) + Filipino (fil),
TURKISH = 33,
UKRAINIAN = 34,
HINDI = 35,
MACEDONIAN = 36,
BENGALI = 37,
INDONESIAN = 38,
LATIN = 39, // UI only.
MALAY = 40,
MALAYALAM = 41,
WELSH = 42, // UI only.
NEPALI = 43,
TELUGU = 44,
ALBANIAN = 45,
TAMIL = 46,
BELARUSIAN = 47,
JAVANESE = 48, // UI only.
OCCITAN = 49, // UI only.
URDU = 50,
BIHARI = 51,
GUJARATI = 52,
THAI = 53,
ARABIC = 54,
CATALAN = 55,
ESPERANTO = 56,
BASQUE = 57,
INTERLINGUA = 58, // UI only.
KANNADA = 59,
PUNJABI = 60,
SCOTS_GAELIC = 61, // UI only.
SWAHILI = 62,
SLOVENIAN = 63,
MARATHI = 64,
MALTESE = 65,
VIETNAMESE = 66,
FRISIAN = 67, // UI only.
SLOVAK = 68,
CHINESE_T = 69, // This is added to solve the problem of
// distinguishing Traditional and Simplified
// Chinese when the encoding is UTF8.
FAROESE = 70, // UI only.
SUNDANESE = 71, // UI only.
UZBEK = 72,
AMHARIC = 73,
AZERBAIJANI = 74,
GEORGIAN = 75,
TIGRINYA = 76, // UI only.
PERSIAN = 77,
BOSNIAN = 78, // UI only. LangId language: CROATIAN (28)
SINHALESE = 79,
NORWEGIAN_N = 80, // UI only. LangId language: NORWEGIAN (10)
PORTUGUESE_P = 81, // UI only. LangId language: PORTUGUESE (12)
PORTUGUESE_B = 82, // UI only. LangId language: PORTUGUESE (12)
XHOSA = 83, // UI only.
ZULU = 84, // UI only.
GUARANI = 85,
SESOTHO = 86, // UI only.
TURKMEN = 87, // UI only.
KYRGYZ = 88,
BRETON = 89, // UI only.
TWI = 90, // UI only.
YIDDISH = 91, // UI only.
SERBO_CROATIAN= 92, // UI only. LangId language: SERBIAN (29)
SOMALI = 93, // UI only.
UIGHUR = 94,
KURDISH = 95,
MONGOLIAN = 96,
ARMENIAN = 97,
LAOTHIAN = 98,
SINDHI = 99,
RHAETO_ROMANCE= 100, // UI only.
AFRIKAANS = 101,
LUXEMBOURGISH = 102, // UI only.
BURMESE = 103,
KHMER = 104,
TIBETAN = 105,
DHIVEHI = 106, // sometimes spelled Divehi, lang of Maldives
CHEROKEE = 107,
SYRIAC = 108, // UI only.
LIMBU = 109, // UI only.
ORIYA = 110,
ASSAMESE = 111, // UI only.
CORSICAN = 112, // UI only.
INTERLINGUE = 113, // UI only.
KAZAKH = 114,
LINGALA = 115, // UI only.
MOLDAVIAN = 116, // UI only. LangId language: ROMANIAN (22)
PASHTO = 117,
QUECHUA = 118, // UI only.
SHONA = 119, // UI only.
TAJIK = 120,
TATAR = 121, // UI only.
TONGA = 122, // UI only.
YORUBA = 123, // UI only.
CREOLES_AND_PIDGINS_ENGLISH_BASED = 124, // UI only.
CREOLES_AND_PIDGINS_FRENCH_BASED = 125, // UI only.
CREOLES_AND_PIDGINS_PORTUGUESE_BASED = 126, // UI only.
CREOLES_AND_PIDGINS_OTHER = 127, // UI only.
MAORI = 128, // UI only.
WOLOF = 129, // UI only.
ABKHAZIAN = 130, // UI only.
AFAR = 131, // UI only.
AYMARA = 132, // UI only.
BASHKIR = 133, // UI only.
BISLAMA = 134, // UI only.
DZONGKHA = 135, // UI only.
FIJIAN = 136, // UI only.
GREENLANDIC = 137, // UI only.
HAUSA = 138, // UI only.
HAITIAN_CREOLE= 139, // UI only.
INUPIAK = 140, // UI only.
INUKTITUT = 141,
KASHMIRI = 142, // UI only.
KINYARWANDA = 143, // UI only.
MALAGASY = 144, // UI only.
NAURU = 145, // UI only.
OROMO = 146, // UI only.
RUNDI = 147, // UI only.
SAMOAN = 148, // UI only.
SANGO = 149, // UI only.
SANSKRIT = 150,
SISWANT = 151, // UI only.
TSONGA = 152, // UI only.
TSWANA = 153, // UI only.
VOLAPUK = 154, // UI only.
ZHUANG = 155, // UI only.
KHASI = 156, // UI only.
SCOTS = 157, // UI only.
GANDA = 158, // UI only.
MANX = 159, // UI only.
MONTENEGRIN = 160, // UI only. LangId language: SERBIAN (29)
NUM_LANGUAGES = 161, // Always keep this at the end. It is not a
// valid Language enum. It is only used to
// indicate the total number of Languages.
// NOTE: If you add a language, you will break a unittest. See the note
// at the top of this enum.
};
#endif // UTIL_LANGUAGES_LANGUAGES_PB_H_

25
app/src/main/jni/util/logging.h
Unescape View File

@ -0,0 +1,25 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_LOGGING_H_
#define UTIL_LOGGING_H_
#undef CHECK
#define CHECK(expr)
#undef DCHECK
#define DCHECK(expr)
#endif // UTIL_LOGGING_H_

53
app/src/main/jni/util/port.h
Unescape View File

@ -0,0 +1,53 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_PORT_H_
#define UTIL_PORT_H_
#include <stdarg.h>
#if defined(_MSC_VER)
#define GG_LONGLONG(x) x##I64
#define GG_ULONGLONG(x) x##UI64
#else
#define GG_LONGLONG(x) x##LL
#define GG_ULONGLONG(x) x##ULL
#endif
// Per C99 7.8.14, define __STDC_CONSTANT_MACROS before including <stdint.h>
// to get the INTn_C and UINTn_C macros for integer constants. It's difficult
// to guarantee any specific ordering of header includes, so it's difficult to
// guarantee that the INTn_C macros can be defined by including <stdint.h> at
// any specific point. Provide GG_INTn_C macros instead.
#define GG_INT8_C(x) (x)
#define GG_INT16_C(x) (x)
#define GG_INT32_C(x) (x)
#define GG_INT64_C(x) GG_LONGLONG(x)
#define GG_UINT8_C(x) (x ## U)
#define GG_UINT16_C(x) (x ## U)
#define GG_UINT32_C(x) (x ## U)
#define GG_UINT64_C(x) GG_ULONGLONG(x)
// Define an OS-neutral wrapper for shared library entry points
#if defined(_WIN32)
#define API_CALL __stdcall
#else
#define API_CALL
#endif
#endif // UTIL_PORT_H_

61
app/src/main/jni/util/string_util.h
Unescape View File

@ -0,0 +1,61 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_STRING_UTIL_H_
#define UTIL_STRING_UTIL_H_
#include <string.h>
namespace base {
#if defined(_WIN32)
// Compare the two strings s1 and s2 without regard to case using
// the current locale; returns 0 if they are equal, 1 if s1 > s2, and -1 if
// s2 > s1 according to a lexicographic comparison.
inline int strcasecmp(const char* s1, const char* s2) {
return _stricmp(s1, s2);
}
inline int strncasecmp(const char* s1, const char* s2, size_t n) {
return _strnicmp(s1, s2, n);
}
#else
inline int strcasecmp(const char* s1, const char* s2) {
return ::strcasecmp(s1, s2);
}
inline int strncasecmp(const char* s1, const char* s2, size_t n) {
return ::strncasecmp(s1, s2, n);
}
#endif
}
#ifndef HAVE_MEMRCHR
#if defined(__GLIBC__) && ((__GLIBC__ > 2) || ((__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 2)))
#define HAVE_MEMRCHR
#endif
#endif
#ifndef HAVE_MEMRCHR
inline void* memrchr(const void* s, int c, size_t n) {
const unsigned char* p = (const unsigned char*) s;
for (p += n; n > 0; n--) {
if (*--p == c)
return (void*) p;
}
return NULL;
}
#endif
#endif // UTIL_STRING_UTIL_H_

66
app/src/main/jni/util/varsetter.h
Unescape View File

@ -0,0 +1,66 @@
// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef UTIL_VARSETTER_H_
#define UTIL_VARSETTER_H_
//
// Use a VarSetter object to temporarily set an object of some sort to
// a particular value. When the VarSetter object is destructed, the
// underlying object will revert to its former value.
//
// Sample code:
//
#if 0
{
bool b = true;
{
VarSetter<bool> bool_setter(&b, false);
// Now b == false.
}
// Now b == true again.
}
#endif
template <class C>
class VarSetter {
public:
// Constructor that just sets the object to a fixed value
VarSetter(C* object, const C& value) : object_(object), old_value_(*object) {
*object = value;
}
~VarSetter() { *object_ = old_value_; }
private:
C*const object_;
C old_value_;
// Disallow
VarSetter(const VarSetter&);
VarSetter& operator=(const VarSetter&);
// VarSetters always live on the stack
static void* operator new (size_t);
static void* operator new[](size_t); // Redundant, no default ctor
static void operator delete (void*);
static void operator delete[](void*);
};
#endif // UTIL_VARSETTER_H_
Write Preview
Loading…
Cancel
Save