优化后端

This commit is contained in:
Miayui
2026-09-11 17:56:56 +08:00
parent 7eed11b774
commit 812e67422d
7299 changed files with 1744959 additions and 112 deletions
+24
View File
@@ -0,0 +1,24 @@
'use strict';
const createWorker = require('./createWorker');
const recognize = async (image, langs, options) => {
const worker = await createWorker(langs, 1, options);
return worker.recognize(image)
.finally(async () => {
await worker.terminate();
});
};
const detect = async (image, options) => {
const worker = await createWorker('osd', 0, options);
return worker.detect(image)
.finally(async () => {
await worker.terminate();
});
};
module.exports = {
recognize,
detect,
};
@@ -0,0 +1,14 @@
'use strict';
/*
* OEM = OCR Engine Mode, and there are 4 possible modes.
*
* By default tesseract.js uses LSTM_ONLY mode.
*
*/
module.exports = {
TESSERACT_ONLY: 0,
LSTM_ONLY: 1,
TESSERACT_LSTM_COMBINED: 2,
DEFAULT: 3,
};
@@ -0,0 +1,21 @@
'use strict';
/*
* PSM = Page Segmentation Mode
*/
module.exports = {
OSD_ONLY: '0',
AUTO_OSD: '1',
AUTO_ONLY: '2',
AUTO: '3',
SINGLE_COLUMN: '4',
SINGLE_BLOCK_VERT_TEXT: '5',
SINGLE_BLOCK: '6',
SINGLE_LINE: '7',
SINGLE_WORD: '8',
CIRCLE_WORD: '9',
SINGLE_CHAR: '10',
SPARSE_TEXT: '11',
SPARSE_TEXT_OSD: '12',
RAW_LINE: '13',
};
@@ -0,0 +1,11 @@
'use strict';
module.exports = {
/*
* Use BlobURL for worker script by default
* TODO: remove this option
*
*/
workerBlobURL: true,
logger: () => {},
};
@@ -0,0 +1,7 @@
'use strict';
module.exports = {
COLOR: 0,
GREY: 1,
BINARY: 2,
};
@@ -0,0 +1,220 @@
'use strict';
/*
* languages with existing tesseract traineddata
* https://tesseract-ocr.github.io/tessdoc/Data-Files#data-files-for-version-400-november-29-2016
*/
/**
* @typedef {object} Languages
* @property {string} AFR Afrikaans
* @property {string} AMH Amharic
* @property {string} ARA Arabic
* @property {string} ASM Assamese
* @property {string} AZE Azerbaijani
* @property {string} AZE_CYRL Azerbaijani - Cyrillic
* @property {string} BEL Belarusian
* @property {string} BEN Bengali
* @property {string} BOD Tibetan
* @property {string} BOS Bosnian
* @property {string} BUL Bulgarian
* @property {string} CAT Catalan; Valencian
* @property {string} CEB Cebuano
* @property {string} CES Czech
* @property {string} CHI_SIM Chinese - Simplified
* @property {string} CHI_TRA Chinese - Traditional
* @property {string} CHR Cherokee
* @property {string} CYM Welsh
* @property {string} DAN Danish
* @property {string} DEU German
* @property {string} DZO Dzongkha
* @property {string} ELL Greek, Modern (1453-)
* @property {string} ENG English
* @property {string} ENM English, Middle (1100-1500)
* @property {string} EPO Esperanto
* @property {string} EST Estonian
* @property {string} EUS Basque
* @property {string} FAS Persian
* @property {string} FIN Finnish
* @property {string} FRA French
* @property {string} FRK German Fraktur
* @property {string} FRM French, Middle (ca. 1400-1600)
* @property {string} GLE Irish
* @property {string} GLG Galician
* @property {string} GRC Greek, Ancient (-1453)
* @property {string} GUJ Gujarati
* @property {string} HAT Haitian; Haitian Creole
* @property {string} HEB Hebrew
* @property {string} HIN Hindi
* @property {string} HRV Croatian
* @property {string} HUN Hungarian
* @property {string} IKU Inuktitut
* @property {string} IND Indonesian
* @property {string} ISL Icelandic
* @property {string} ITA Italian
* @property {string} ITA_OLD Italian - Old
* @property {string} JAV Javanese
* @property {string} JPN Japanese
* @property {string} KAN Kannada
* @property {string} KAT Georgian
* @property {string} KAT_OLD Georgian - Old
* @property {string} KAZ Kazakh
* @property {string} KHM Central Khmer
* @property {string} KIR Kirghiz; Kyrgyz
* @property {string} KOR Korean
* @property {string} KUR Kurdish
* @property {string} LAO Lao
* @property {string} LAT Latin
* @property {string} LAV Latvian
* @property {string} LIT Lithuanian
* @property {string} MAL Malayalam
* @property {string} MAR Marathi
* @property {string} MKD Macedonian
* @property {string} MLT Maltese
* @property {string} MSA Malay
* @property {string} MYA Burmese
* @property {string} NEP Nepali
* @property {string} NLD Dutch; Flemish
* @property {string} NOR Norwegian
* @property {string} ORI Oriya
* @property {string} PAN Panjabi; Punjabi
* @property {string} POL Polish
* @property {string} POR Portuguese
* @property {string} PUS Pushto; Pashto
* @property {string} RON Romanian; Moldavian; Moldovan
* @property {string} RUS Russian
* @property {string} SAN Sanskrit
* @property {string} SIN Sinhala; Sinhalese
* @property {string} SLK Slovak
* @property {string} SLV Slovenian
* @property {string} SPA Spanish; Castilian
* @property {string} SPA_OLD Spanish; Castilian - Old
* @property {string} SQI Albanian
* @property {string} SRP Serbian
* @property {string} SRP_LATN Serbian - Latin
* @property {string} SWA Swahili
* @property {string} SWE Swedish
* @property {string} SYR Syriac
* @property {string} TAM Tamil
* @property {string} TEL Telugu
* @property {string} TGK Tajik
* @property {string} TGL Tagalog
* @property {string} THA Thai
* @property {string} TIR Tigrinya
* @property {string} TUR Turkish
* @property {string} UIG Uighur; Uyghur
* @property {string} UKR Ukrainian
* @property {string} URD Urdu
* @property {string} UZB Uzbek
* @property {string} UZB_CYRL Uzbek - Cyrillic
* @property {string} VIE Vietnamese
* @property {string} YID Yiddish
*/
/**
* @type {Languages}
*/
module.exports = {
AFR: 'afr',
AMH: 'amh',
ARA: 'ara',
ASM: 'asm',
AZE: 'aze',
AZE_CYRL: 'aze_cyrl',
BEL: 'bel',
BEN: 'ben',
BOD: 'bod',
BOS: 'bos',
BUL: 'bul',
CAT: 'cat',
CEB: 'ceb',
CES: 'ces',
CHI_SIM: 'chi_sim',
CHI_TRA: 'chi_tra',
CHR: 'chr',
CYM: 'cym',
DAN: 'dan',
DEU: 'deu',
DZO: 'dzo',
ELL: 'ell',
ENG: 'eng',
ENM: 'enm',
EPO: 'epo',
EST: 'est',
EUS: 'eus',
FAS: 'fas',
FIN: 'fin',
FRA: 'fra',
FRK: 'frk',
FRM: 'frm',
GLE: 'gle',
GLG: 'glg',
GRC: 'grc',
GUJ: 'guj',
HAT: 'hat',
HEB: 'heb',
HIN: 'hin',
HRV: 'hrv',
HUN: 'hun',
IKU: 'iku',
IND: 'ind',
ISL: 'isl',
ITA: 'ita',
ITA_OLD: 'ita_old',
JAV: 'jav',
JPN: 'jpn',
KAN: 'kan',
KAT: 'kat',
KAT_OLD: 'kat_old',
KAZ: 'kaz',
KHM: 'khm',
KIR: 'kir',
KOR: 'kor',
KUR: 'kur',
LAO: 'lao',
LAT: 'lat',
LAV: 'lav',
LIT: 'lit',
MAL: 'mal',
MAR: 'mar',
MKD: 'mkd',
MLT: 'mlt',
MSA: 'msa',
MYA: 'mya',
NEP: 'nep',
NLD: 'nld',
NOR: 'nor',
ORI: 'ori',
PAN: 'pan',
POL: 'pol',
POR: 'por',
PUS: 'pus',
RON: 'ron',
RUS: 'rus',
SAN: 'san',
SIN: 'sin',
SLK: 'slk',
SLV: 'slv',
SPA: 'spa',
SPA_OLD: 'spa_old',
SQI: 'sqi',
SRP: 'srp',
SRP_LATN: 'srp_latn',
SWA: 'swa',
SWE: 'swe',
SYR: 'syr',
TAM: 'tam',
TEL: 'tel',
TGK: 'tgk',
TGL: 'tgl',
THA: 'tha',
TIR: 'tir',
TUR: 'tur',
UIG: 'uig',
UKR: 'ukr',
URD: 'urd',
UZB: 'uzb',
UZB_CYRL: 'uzb_cyrl',
VIE: 'vie',
YID: 'yid',
};
+23
View File
@@ -0,0 +1,23 @@
'use strict';
const getId = require('./utils/getId');
let jobCounter = 0;
module.exports = ({
id: _id,
action,
payload = {},
}) => {
let id = _id;
if (typeof id === 'undefined') {
id = getId('Job', jobCounter);
jobCounter += 1;
}
return {
id,
action,
payload,
};
};
@@ -0,0 +1,82 @@
'use strict';
const createJob = require('./createJob');
const { log } = require('./utils/log');
const getId = require('./utils/getId');
let schedulerCounter = 0;
module.exports = () => {
const id = getId('Scheduler', schedulerCounter);
const workers = {};
const runningWorkers = {};
let jobQueue = [];
schedulerCounter += 1;
const getQueueLen = () => jobQueue.length;
const getNumWorkers = () => Object.keys(workers).length;
const dequeue = () => {
if (jobQueue.length !== 0) {
const wIds = Object.keys(workers);
for (let i = 0; i < wIds.length; i += 1) {
if (typeof runningWorkers[wIds[i]] === 'undefined') {
jobQueue[0](workers[wIds[i]]);
break;
}
}
}
};
const queue = (action, payload) => (
new Promise((resolve, reject) => {
const job = createJob({ action, payload });
jobQueue.push(async (w) => {
jobQueue.shift();
runningWorkers[w.id] = job;
try {
resolve(await w[action].apply(this, [...payload, job.id]));
} catch (err) {
reject(err);
} finally {
delete runningWorkers[w.id];
dequeue();
}
});
log(`[${id}]: Add ${job.id} to JobQueue`);
log(`[${id}]: JobQueue length=${jobQueue.length}`);
dequeue();
})
);
const addWorker = (w) => {
workers[w.id] = w;
log(`[${id}]: Add ${w.id}`);
log(`[${id}]: Number of workers=${getNumWorkers()}`);
dequeue();
return w.id;
};
const addJob = async (action, ...payload) => {
if (getNumWorkers() === 0) {
throw Error(`[${id}]: You need to have at least one worker before adding jobs`);
}
return queue(action, payload);
};
const terminate = async () => {
Object.keys(workers).forEach(async (wid) => {
await workers[wid].terminate();
});
jobQueue = [];
};
return {
addWorker,
addJob,
terminate,
getQueueLen,
getNumWorkers,
};
};
@@ -0,0 +1,246 @@
'use strict';
const resolvePaths = require('./utils/resolvePaths');
const createJob = require('./createJob');
const { log } = require('./utils/log');
const getId = require('./utils/getId');
const OEM = require('./constants/OEM');
const {
defaultOptions,
spawnWorker,
terminateWorker,
onMessage,
loadImage,
send,
} = require('./worker/node');
let workerCounter = 0;
module.exports = async (langs = 'eng', oem = OEM.LSTM_ONLY, _options = {}, config = {}) => {
const id = getId('Worker', workerCounter);
const {
logger,
errorHandler,
...options
} = resolvePaths({
...defaultOptions,
..._options,
});
const promises = {};
// Current langs, oem, and config file.
// Used if the user ever re-initializes the worker using `worker.reinitialize`.
const currentLangs = typeof langs === 'string' ? langs.split('+') : langs;
let currentOem = oem;
let currentConfig = config;
const lstmOnlyCore = [OEM.DEFAULT, OEM.LSTM_ONLY].includes(oem) && !options.legacyCore;
let workerResReject;
let workerResResolve;
const workerRes = new Promise((resolve, reject) => {
workerResResolve = resolve;
workerResReject = reject;
});
const workerError = (event) => { workerResReject(event.message); };
let worker = spawnWorker(options);
worker.onerror = workerError;
workerCounter += 1;
const startJob = ({ id: jobId, action, payload }) => (
new Promise((resolve, reject) => {
log(`[${id}]: Start ${jobId}, action=${action}`);
// Using both `action` and `jobId` in case user provides non-unique `jobId`.
const promiseId = `${action}-${jobId}`;
promises[promiseId] = { resolve, reject };
send(worker, {
workerId: id,
jobId,
action,
payload,
});
})
);
const load = () => (
console.warn('`load` is depreciated and should be removed from code (workers now come pre-loaded)')
);
const loadInternal = (jobId) => (
startJob(createJob({
id: jobId, action: 'load', payload: { options: { lstmOnly: lstmOnlyCore, corePath: options.corePath, logging: options.logging } },
}))
);
const writeText = (path, text, jobId) => (
startJob(createJob({
id: jobId,
action: 'FS',
payload: { method: 'writeFile', args: [path, text] },
}))
);
const readText = (path, jobId) => (
startJob(createJob({
id: jobId,
action: 'FS',
payload: { method: 'readFile', args: [path, { encoding: 'utf8' }] },
}))
);
const removeFile = (path, jobId) => (
startJob(createJob({
id: jobId,
action: 'FS',
payload: { method: 'unlink', args: [path] },
}))
);
const FS = (method, args, jobId) => (
startJob(createJob({
id: jobId,
action: 'FS',
payload: { method, args },
}))
);
const loadLanguageInternal = (_langs, jobId) => startJob(createJob({
id: jobId,
action: 'loadLanguage',
payload: {
langs: _langs,
options: {
langPath: options.langPath,
dataPath: options.dataPath,
cachePath: options.cachePath,
cacheMethod: options.cacheMethod,
gzip: options.gzip,
lstmOnly: [OEM.DEFAULT, OEM.LSTM_ONLY].includes(currentOem)
&& !options.legacyLang,
},
},
}));
const initializeInternal = (_langs, _oem, _config, jobId) => (
startJob(createJob({
id: jobId,
action: 'initialize',
payload: { langs: _langs, oem: _oem, config: _config },
}))
);
const reinitialize = (langs = 'eng', oem, config, jobId) => { // eslint-disable-line
if (lstmOnlyCore && [OEM.TESSERACT_ONLY, OEM.TESSERACT_LSTM_COMBINED].includes(oem)) throw Error('Legacy model requested but code missing.');
const _oem = oem || currentOem;
currentOem = _oem;
const _config = config || currentConfig;
currentConfig = _config;
// Only load langs that are not already loaded.
// This logic fails if the user downloaded the LSTM-only English data for a language
// and then uses `worker.reinitialize` to switch to the Legacy engine.
// However, the correct data will still be downloaded after initialization fails
// and this can be avoided entirely if the user loads the correct data ahead of time.
const langsArr = typeof langs === 'string' ? langs.split('+') : langs;
const _langs = langsArr.filter((x) => !currentLangs.includes(x));
currentLangs.push(..._langs);
if (_langs.length > 0) {
return loadLanguageInternal(_langs, jobId)
.then(() => initializeInternal(langs, _oem, _config, jobId));
}
return initializeInternal(langs, _oem, _config, jobId);
};
const setParameters = (params = {}, jobId) => (
startJob(createJob({
id: jobId,
action: 'setParameters',
payload: { params },
}))
);
const recognize = async (image, opts = {}, output = {
text: true,
}, jobId) => (
startJob(createJob({
id: jobId,
action: 'recognize',
payload: { image: await loadImage(image), options: opts, output },
}))
);
const detect = async (image, jobId) => {
if (lstmOnlyCore) throw Error('`worker.detect` requires Legacy model, which was not loaded.');
return startJob(createJob({
id: jobId,
action: 'detect',
payload: { image: await loadImage(image) },
}));
};
const terminate = async () => {
if (worker !== null) {
/*
await startJob(createJob({
id: jobId,
action: 'terminate',
}));
*/
terminateWorker(worker);
worker = null;
}
return Promise.resolve();
};
onMessage(worker, ({
workerId, jobId, status, action, data,
}) => {
const promiseId = `${action}-${jobId}`;
if (status === 'resolve') {
log(`[${workerId}]: Complete ${jobId}`);
promises[promiseId].resolve({ jobId, data });
delete promises[promiseId];
} else if (status === 'reject') {
promises[promiseId].reject(data);
delete promises[promiseId];
if (action === 'load') workerResReject(data);
if (errorHandler) {
errorHandler(data);
} else {
throw Error(data);
}
} else if (status === 'progress') {
logger({ ...data, userJobId: jobId });
}
});
const resolveObj = {
id,
worker,
load,
writeText,
readText,
removeFile,
FS,
reinitialize,
setParameters,
recognize,
detect,
terminate,
};
loadInternal()
.then(() => loadLanguageInternal(langs))
.then(() => initializeInternal(langs, oem, config))
.then(() => workerResResolve(resolveObj))
.catch(() => {});
return workerRes;
};
+234
View File
@@ -0,0 +1,234 @@
declare namespace Tesseract {
function createScheduler(): Scheduler
function createWorker(langs?: string | string[] | Lang[], oem?: OEM, options?: Partial<WorkerOptions>, config?: string | Partial<InitOptions>): Promise<Worker>
function setLogging(logging: boolean): void
function recognize(image: ImageLike, langs?: string, options?: Partial<WorkerOptions>): Promise<RecognizeResult>
function detect(image: ImageLike, options?: Partial<WorkerOptions>): any
interface Scheduler {
addWorker(worker: Worker): string
addJob(action: 'recognize', ...args: Parameters<Worker['recognize']>): Promise<RecognizeResult>
addJob(action: 'detect', ...args: Parameters<Worker['detect']>): Promise<DetectResult>
terminate(): Promise<any>
getQueueLen(): number
getNumWorkers(): number
}
interface Worker {
load(jobId?: string): Promise<ConfigResult>
writeText(path: string, text: string, jobId?: string): Promise<ConfigResult>
readText(path: string, jobId?: string): Promise<ConfigResult>
removeText(path: string, jobId?: string): Promise<ConfigResult>
FS(method: string, args: any[], jobId?: string): Promise<ConfigResult>
reinitialize(langs?: string | Lang[], oem?: OEM, config?: string | Partial<InitOptions>, jobId?: string): Promise<ConfigResult>
setParameters(params: Partial<WorkerParams>, jobId?: string): Promise<ConfigResult>
getImage(type: imageType): string
recognize(image: ImageLike, options?: Partial<RecognizeOptions>, output?: Partial<OutputFormats>, jobId?: string): Promise<RecognizeResult>
detect(image: ImageLike, jobId?: string): Promise<DetectResult>
terminate(jobId?: string): Promise<ConfigResult>
}
interface Lang {
code: string;
data: unknown;
}
interface InitOptions {
load_system_dawg: string
load_freq_dawg: string
load_unambig_dawg: string
load_punc_dawg: string
load_number_dawg: string
load_bigram_dawg: string
}
type LoggerMessage = {
jobId: string
progress: number
status: string
userJobId: string
workerId: string
}
interface WorkerOptions {
corePath: string
langPath: string
cachePath: string
dataPath: string
workerPath: string
cacheMethod: string
workerBlobURL: boolean
gzip: boolean
legacyLang: boolean
legacyCore: boolean
logger: (arg: LoggerMessage) => void,
errorHandler: (arg: any) => void
}
interface WorkerParams {
tessedit_pageseg_mode: PSM
tessedit_char_whitelist: string
tessedit_char_blacklist: string
preserve_interword_spaces: string
user_defined_dpi: string
[propName: string]: any
}
interface OutputFormats {
text: boolean;
blocks: boolean;
layoutBlocks: boolean;
hocr: boolean;
tsv: boolean;
box: boolean;
unlv: boolean;
osd: boolean;
pdf: boolean;
imageColor: boolean;
imageGrey: boolean;
imageBinary: boolean;
debug: boolean;
}
interface RecognizeOptions {
rectangle: Rectangle
pdfTitle: string
pdfTextOnly: boolean
rotateAuto: boolean
rotateRadians: number
}
interface ConfigResult {
jobId: string
data: any
}
interface RecognizeResult {
jobId: string
data: Page
}
interface DetectResult {
jobId: string
data: DetectData
}
interface DetectData {
tesseract_script_id: number | null
script: string | null
script_confidence: number | null
orientation_degrees: number | null
orientation_confidence: number | null
}
interface Rectangle {
left: number
top: number
width: number
height: number
}
enum OEM {
TESSERACT_ONLY,
LSTM_ONLY,
TESSERACT_LSTM_COMBINED,
DEFAULT,
}
enum PSM {
OSD_ONLY = '0',
AUTO_OSD = '1',
AUTO_ONLY = '2',
AUTO = '3',
SINGLE_COLUMN = '4',
SINGLE_BLOCK_VERT_TEXT = '5',
SINGLE_BLOCK = '6',
SINGLE_LINE = '7',
SINGLE_WORD = '8',
CIRCLE_WORD = '9',
SINGLE_CHAR = '10',
SPARSE_TEXT = '11',
SPARSE_TEXT_OSD = '12',
RAW_LINE = '13'
}
const enum imageType {
COLOR = 0,
GREY = 1,
BINARY = 2
}
type ImageLike = string | HTMLImageElement | HTMLCanvasElement | HTMLVideoElement
| CanvasRenderingContext2D | File | Blob | Buffer | OffscreenCanvas;
interface Block {
paragraphs: Paragraph[];
text: string;
confidence: number;
bbox: Bbox;
blocktype: string;
page: Page;
}
interface Baseline {
x0: number;
y0: number;
x1: number;
y1: number;
}
interface RowAttributes {
ascenders: number;
descenders: number;
rowHeight: number;
}
interface Bbox {
x0: number;
y0: number;
x1: number;
y1: number;
}
interface Line {
words: Word[];
text: string;
confidence: number;
baseline: Baseline;
rowAttributes: RowAttributes
bbox: Bbox;
}
interface Paragraph {
lines: Line[];
text: string;
confidence: number;
bbox: Bbox;
is_ltr: boolean;
}
interface Symbol {
text: string;
confidence: number;
bbox: Bbox;
is_superscript: boolean;
is_subscript: boolean;
is_dropcap: boolean;
}
interface Choice {
text: string;
confidence: number;
}
interface Word {
symbols: Symbol[];
choices: Choice[];
text: string;
confidence: number;
bbox: Bbox;
font_name: string;
}
interface Page {
blocks: Block[] | null;
confidence: number;
oem: string;
osd: string;
psm: string;
text: string;
version: string;
hocr: string | null;
tsv: string | null;
box: string | null;
unlv: string | null;
sd: string | null;
imageColor: string | null;
imageGrey: string | null;
imageBinary: string | null;
rotateRadians: number | null;
pdf: number[] | null;
debug: string | null;
}
}
export = Tesseract;
export as namespace Tesseract;
+29
View File
@@ -0,0 +1,29 @@
'use strict';
/**
*
* Entry point for tesseract.js, should be the entry when bundling.
*
* @fileoverview entry point for tesseract.js
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
require('regenerator-runtime/runtime');
const createScheduler = require('./createScheduler');
const createWorker = require('./createWorker');
const Tesseract = require('./Tesseract');
const languages = require('./constants/languages');
const OEM = require('./constants/OEM');
const PSM = require('./constants/PSM');
const { setLogging } = require('./utils/log');
module.exports = {
languages,
OEM,
PSM,
createScheduler,
createWorker,
setLogging,
...Tesseract,
};
@@ -0,0 +1,19 @@
'use strict';
module.exports = (key) => {
const env = {};
if (typeof WorkerGlobalScope !== 'undefined') {
env.type = 'webworker';
} else if (typeof document === 'object') {
env.type = 'browser';
} else if (typeof process === 'object' && typeof require === 'function') {
env.type = 'node';
}
if (typeof key === 'undefined') {
return env;
}
return env[key];
};
@@ -0,0 +1,5 @@
'use strict';
module.exports = (prefix, cnt) => (
`${prefix}-${cnt}-${Math.random().toString(16).slice(3, 8)}`
);
+11
View File
@@ -0,0 +1,11 @@
'use strict';
let logging = false;
exports.logging = logging;
exports.setLogging = (_logging) => {
logging = _logging;
};
exports.log = (...args) => (logging ? console.log.apply(this, args) : null);
@@ -0,0 +1,15 @@
'use strict';
const isBrowser = require('./getEnvironment')('type') === 'browser';
const resolveURL = isBrowser ? s => (new URL(s, window.location.href)).href : s => s; // eslint-disable-line
module.exports = (options) => {
const opts = { ...options };
['corePath', 'workerPath', 'langPath'].forEach((key) => {
if (options[key]) {
opts[key] = resolveURL(opts[key]);
}
});
return opts;
};
@@ -0,0 +1,12 @@
'use strict';
const { set, get, del } = require('idb-keyval');
module.exports = {
readCache: get,
writeCache: set,
deleteCache: del,
checkCache: (path) => (
get(path).then((v) => typeof v !== 'undefined')
),
};
@@ -0,0 +1,59 @@
'use strict';
const { simd, relaxedSimd } = require('wasm-feature-detect');
const coreVersion = require('../../../package.json').dependencies['tesseract.js-core'];
module.exports = async (lstmOnly, corePath, res) => {
if (typeof global.TesseractCore === 'undefined') {
const statusText = 'loading tesseract core';
res.progress({ status: statusText, progress: 0 });
// If the user specifies a core path, we use that
// Otherwise, default to CDN
const corePathImport = corePath || `https://cdn.jsdelivr.net/npm/tesseract.js-core@v${coreVersion.substring(1)}`;
// If a user specifies a specific JavaScript file, load that file.
// Otherwise, assume a directory has been provided, and load either
// tesseract-core.wasm.js or tesseract-core-simd.wasm.js depending
// on whether this device has SIMD support.
let corePathImportFile;
if (corePathImport.slice(-2) === 'js') {
corePathImportFile = corePathImport;
} else {
const simdSupport = await simd();
const relaxedSimdSupport = await relaxedSimd();
if (relaxedSimdSupport) {
if (lstmOnly) {
corePathImportFile = `${corePathImport.replace(/\/$/, '')}/tesseract-core-relaxedsimd-lstm.wasm.js`;
} else {
corePathImportFile = `${corePathImport.replace(/\/$/, '')}/tesseract-core-relaxedsimd.wasm.js`;
}
} else if (simdSupport) {
if (lstmOnly) {
corePathImportFile = `${corePathImport.replace(/\/$/, '')}/tesseract-core-simd-lstm.wasm.js`;
} else {
corePathImportFile = `${corePathImport.replace(/\/$/, '')}/tesseract-core-simd.wasm.js`;
}
} else if (lstmOnly) {
corePathImportFile = `${corePathImport.replace(/\/$/, '')}/tesseract-core-lstm.wasm.js`;
} else {
corePathImportFile = `${corePathImport.replace(/\/$/, '')}/tesseract-core.wasm.js`;
}
}
// Create a module named `global.TesseractCore`
global.importScripts(corePathImportFile);
// Tesseract.js-core versions through 4.0.3 create a module named `global.TesseractCoreWASM`,
// so we account for that here to preserve backwards compatibility.
// This part can be removed when Tesseract.js-core v4.0.3 becomes incompatible for other reasons
if (typeof global.TesseractCore === 'undefined' && typeof global.TesseractCoreWASM !== 'undefined' && typeof WebAssembly === 'object') {
global.TesseractCore = global.TesseractCoreWASM;
} else if (typeof global.TesseractCore === 'undefined') {
throw Error('Failed to load TesseractCore');
}
res.progress({ status: statusText, progress: 1 });
}
return global.TesseractCore;
};
@@ -0,0 +1,3 @@
'use strict';
module.exports = require('zlibjs').gunzipSync;
@@ -0,0 +1,34 @@
'use strict';
/**
*
* Browser worker scripts
*
* @fileoverview Browser worker implementation
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
const worker = require('..');
const getCore = require('./getCore');
const gunzip = require('./gunzip');
const cache = require('./cache');
/*
* register message handler
*/
global.addEventListener('message', ({ data }) => {
worker.dispatchHandlers(data, (obj) => postMessage(obj));
});
/*
* getCore is a sync function to load and return
* TesseractCore.
*/
worker.setAdapter({
getCore,
gunzip,
fetch: () => {},
...cache,
});
@@ -0,0 +1,21 @@
'use strict';
/*
* default output formats for tesseract.js
*/
module.exports = {
text: true,
blocks: false,
layoutBlocks: false,
hocr: false,
tsv: false,
box: false,
unlv: false,
osd: false,
pdf: false,
imageColor: false,
imageGrey: false,
imageBinary: false,
debug: false,
};
@@ -0,0 +1,543 @@
'use strict';
/**
*
* Worker script for browser and node
*
* @fileoverview Worker script for browser and node
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
require('regenerator-runtime/runtime');
const isURL = require('is-url');
const dump = require('./utils/dump');
const env = require('../utils/getEnvironment')('type');
const setImage = require('./utils/setImage');
const defaultOutput = require('./constants/defaultOutput');
const { log, setLogging } = require('../utils/log');
const PSM = require('../constants/PSM');
/*
* Tesseract Module returned by TesseractCore.
*/
let TessModule;
/*
* TessearctBaseAPI instance
*/
let api = null;
let latestJob;
let adapter = {};
let params = {};
let loadLanguageLangsWorker;
let loadLanguageOptionsWorker;
let dataFromCache = false;
const load = async ({ workerId, jobId, payload: { options: { lstmOnly, corePath, logging } } }, res) => { // eslint-disable-line max-len
setLogging(logging);
const statusText = 'initializing tesseract';
if (!TessModule) {
const Core = await adapter.getCore(lstmOnly, corePath, res);
res.progress({ workerId, status: statusText, progress: 0 });
Core({
TesseractProgress(percent) {
latestJob.progress({
workerId,
jobId,
status: 'recognizing text',
progress: Math.max(0, (percent - 30) / 70),
});
},
}).then((tessModule) => {
TessModule = tessModule;
res.progress({ workerId, status: statusText, progress: 1 });
res.resolve({ loaded: true });
});
} else {
res.resolve({ loaded: true });
}
};
const FS = async ({ workerId, payload: { method, args } }, res) => {
log(`[${workerId}]: FS.${method}`);
res.resolve(TessModule.FS[method](...args));
};
const loadLanguage = async ({
workerId,
payload: {
langs,
options: {
langPath,
dataPath,
cachePath,
cacheMethod,
gzip = true,
lstmOnly,
},
},
},
res) => {
// Remember options for later, as cache may be deleted if `initialize` fails
loadLanguageLangsWorker = langs;
loadLanguageOptionsWorker = {
langPath,
dataPath,
cachePath,
cacheMethod,
gzip,
lstmOnly,
};
const statusText = 'loading language traineddata';
const langsArr = typeof langs === 'string' ? langs.split('+') : langs;
let progress = 0;
const loadAndGunzipFile = async (_lang) => {
const lang = typeof _lang === 'string' ? _lang : _lang.code;
const readCache = ['refresh', 'none'].includes(cacheMethod)
? () => Promise.resolve()
: adapter.readCache;
let data = null;
let newData = false;
// Check for existing .traineddata file in cache
// This automatically fails if cacheMethod is set to 'refresh' or 'none'
try {
const _data = await readCache(`${cachePath || '.'}/${lang}.traineddata`);
if (typeof _data !== 'undefined') {
log(`[${workerId}]: Load ${lang}.traineddata from cache`);
data = _data;
dataFromCache = true;
} else {
throw Error('Not found in cache');
}
// Attempt to fetch new .traineddata file
} catch (e) {
newData = true;
log(`[${workerId}]: Load ${lang}.traineddata from ${langPath}`);
if (typeof _lang === 'string') {
let path = null;
// If `langPath` if not explicitly set by the user, the jsdelivr CDN is used.
// Data supporting the Legacy model is only included if `lstmOnly` is not true.
// This saves a significant amount of data for the majority of users that use LSTM only.
const langPathDownload = langPath || (lstmOnly ? `https://cdn.jsdelivr.net/npm/@tesseract.js-data/${lang}/4.0.0_best_int` : `https://cdn.jsdelivr.net/npm/@tesseract.js-data/${lang}/4.0.0`);
// For Node.js, langPath may be a URL or local file path
// The is-url package is used to tell the difference
// For the browser version, langPath is assumed to be a URL
if (env !== 'node' || isURL(langPathDownload) || langPathDownload.startsWith('moz-extension://') || langPathDownload.startsWith('chrome-extension://') || langPathDownload.startsWith('file://')) { /** When langPathDownload is an URL */
path = langPathDownload.replace(/\/$/, '');
}
// langPathDownload is a URL, fetch from server
if (path !== null) {
const fetchUrl = `${path}/${lang}.traineddata${gzip ? '.gz' : ''}`;
const resp = await (env === 'webworker' ? fetch : adapter.fetch)(fetchUrl);
if (!resp.ok) {
throw Error(`Network error while fetching ${fetchUrl}. Response code: ${resp.status}`);
}
data = new Uint8Array(await resp.arrayBuffer());
// langPathDownload is a local file, read .traineddata from local filesystem
// (adapter.readCache is a generic file read function in Node.js version)
} else {
data = await adapter.readCache(`${langPathDownload}/${lang}.traineddata${gzip ? '.gz' : ''}`);
}
} else {
data = _lang.data; // eslint-disable-line
}
}
progress += 0.5 / langsArr.length;
if (res) res.progress({ workerId, status: statusText, progress });
// Check for gzip magic numbers (1F and 8B in hex)
const isGzip = (data[0] === 31 && data[1] === 139) || (data[1] === 31 && data[0] === 139);
if (isGzip) {
data = adapter.gunzip(data);
}
if (TessModule) {
if (dataPath) {
try {
TessModule.FS.mkdir(dataPath);
} catch (err) {
if (res) res.reject(err.toString());
}
}
TessModule.FS.writeFile(`${dataPath || '.'}/${lang}.traineddata`, data);
}
if (newData && ['write', 'refresh', undefined].includes(cacheMethod)) {
try {
await adapter.writeCache(`${cachePath || '.'}/${lang}.traineddata`, data);
} catch (err) {
log(`[${workerId}]: Failed to write ${lang}.traineddata to cache due to error:`);
log(err.toString());
}
}
progress += 0.5 / langsArr.length;
// Make sure last progress message is 1 (not 0.9999)
if (Math.round(progress * 100) === 100) progress = 1;
if (res) res.progress({ workerId, status: statusText, progress });
};
if (res) res.progress({ workerId, status: statusText, progress: 0 });
try {
await Promise.all(langsArr.map(loadAndGunzipFile));
if (res) res.resolve(langs);
} catch (err) {
if (res) res.reject(err.toString());
}
};
const setParameters = async ({ payload: { params: _params } }, res) => {
// A small number of parameters can only be set at initialization.
// These can only be set using (1) the `oem` argument of `initialize` (for setting the oem)
// or (2) the `config` argument of `initialize` (for all other settings).
// Attempting to set these using this function will have no impact so a warning is printed.
// This list is generated by searching the Tesseract codebase for parameters
// defined with `[type]_INIT_MEMBER` rather than `[type]_MEMBER`.
const initParamNames = ['ambigs_debug_level', 'user_words_suffix', 'user_patterns_suffix', 'user_patterns_suffix',
'load_system_dawg', 'load_freq_dawg', 'load_unambig_dawg', 'load_punc_dawg', 'load_number_dawg', 'load_bigram_dawg',
'tessedit_ocr_engine_mode', 'tessedit_init_config_only', 'language_model_ngram_on', 'language_model_use_sigmoidal_certainty'];
const initParamStr = Object.keys(_params)
.filter((k) => initParamNames.includes(k))
.join(', ');
if (initParamStr.length > 0) console.log(`Attempted to set parameters that can only be set during initialization: ${initParamStr}`);
Object.keys(_params)
.filter((k) => !k.startsWith('tessjs_'))
.forEach((key) => {
api.SetVariable(key, _params[key]);
});
params = { ...params, ..._params };
if (typeof res !== 'undefined') {
res.resolve(params);
}
};
const initialize = async ({
workerId,
payload: { langs: _langs, oem, config },
}, res) => {
const langs = (typeof _langs === 'string')
? _langs
: _langs.map((l) => ((typeof l === 'string') ? l : l.data)).join('+');
const statusText = 'initializing api';
try {
res.progress({
workerId, status: statusText, progress: 0,
});
if (api !== null) {
api.End();
}
let configFile;
let configStr;
// config argument may either be config file text, or object with key/value pairs
// In the latter case we convert to config file text here
if (config && typeof config === 'object' && Object.keys(config).length > 0) {
configStr = JSON.stringify(config).replace(/,/g, '\n').replace(/:/g, ' ').replace(/["'{}]/g, '');
} else if (config && typeof config === 'string') {
configStr = config;
}
if (typeof configStr === 'string') {
configFile = '/config';
TessModule.FS.writeFile(configFile, configStr);
}
api = new TessModule.TessBaseAPI();
let status = api.Init(null, langs, oem, configFile);
if (status === -1) {
// Cache is deleted if initialization fails to avoid keeping bad data in cache
// This assumes that initialization failing only occurs due to bad .traineddata,
// this should be refined if other reasons for init failing are encountered.
// The "if" condition skips this section if either (1) cache is disabled [so the issue
// is definitely unrelated to cached data] or (2) cache is set to read-only
// [so we do not have permission to make any changes].
if (['write', 'refresh', undefined].includes(loadLanguageOptionsWorker.cacheMethod)) {
const langsArr = langs.split('+');
const delCachePromise = langsArr.map((lang) => adapter.deleteCache(`${loadLanguageOptionsWorker.cachePath || '.'}/${lang}.traineddata`));
await Promise.all(delCachePromise);
// Check for the case when (1) data was loaded from the cache and
// (2) the data does not support the requested OEM.
// In this case, loadLanguage is re-run and initialization is attempted a second time.
// This is because `loadLanguage` has no mechanism for checking whether the cached data
// supports the requested model, so this only becomes apparent when initialization fails.
// Check for this error message:
// eslint-disable-next-line
// "Tesseract (legacy) engine requested, but components are not present in ./eng.traineddata!!""
// The .wasm build of Tesseract saves this message in a separate file
// (in addition to the normal debug file location).
const debugStr = TessModule.FS.readFile('/debugDev.txt', { encoding: 'utf8', flags: 'a+' });
if (dataFromCache && /components are not present/.test(debugStr)) {
log('Data from cache missing requested OEM model. Attempting to refresh cache with new language data.');
// In this case, language data is re-loaded
await loadLanguage({ workerId, payload: { langs: loadLanguageLangsWorker, options: loadLanguageOptionsWorker } }); // eslint-disable-line max-len
status = api.Init(null, langs, oem, configFile);
if (status === -1) {
log('Language data refresh failed.');
const delCachePromise2 = langsArr.map((lang) => adapter.deleteCache(`${loadLanguageOptionsWorker.cachePath || '.'}/${lang}.traineddata`));
await Promise.all(delCachePromise2);
} else {
log('Language data refresh successful.');
}
}
}
}
if (status === -1) {
res.reject('initialization failed');
}
res.progress({
workerId, status: statusText, progress: 1,
});
res.resolve();
} catch (err) {
res.reject(err.toString());
}
};
// Combines default output with user-specified options and
// counts (1) total output formats requested and (2) outputs that require OCR
const processOutput = (output) => {
const workingOutput = JSON.parse(JSON.stringify(defaultOutput));
const nonRecOutputs = ['imageColor', 'imageGrey', 'imageBinary', 'layoutBlocks', 'debug'];
let recOutputCount = 0;
for (const prop of Object.keys(output)) {
workingOutput[prop] = output[prop];
}
for (const prop of Object.keys(workingOutput)) {
if (workingOutput[prop]) {
if (!nonRecOutputs.includes(prop)) {
recOutputCount += 1;
}
}
}
const skipRecognition = recOutputCount === 0;
return { workingOutput, skipRecognition };
};
// List of options for Tesseract.js (rather than passed through to Tesseract),
// not including those with prefix "tessjs_"
const tessjsOptions = ['rectangle', 'pdfTitle', 'pdfTextOnly', 'rotateAuto', 'rotateRadians'];
const recognize = async ({
payload: {
image, options, output,
},
}, res) => {
try {
const optionsTess = {};
if (typeof options === 'object' && Object.keys(options).length > 0) {
// The options provided by users contain a mix of options for Tesseract.js
// and parameters passed through to Tesseract.
for (const param of Object.keys(options)) {
if (!param.startsWith('tessjs_') && !tessjsOptions.includes(param)) {
optionsTess[param] = options[param];
}
}
}
if (output.debug) {
optionsTess.debug_file = '/debugInternal.txt';
TessModule.FS.writeFile('/debugInternal.txt', '');
}
// If any parameters are changed here they are changed back at the end
if (Object.keys(optionsTess).length > 0) {
api.SaveParameters();
for (const prop of Object.keys(optionsTess)) {
api.SetVariable(prop, optionsTess[prop]);
}
}
const { workingOutput, skipRecognition } = processOutput(output);
// When the auto-rotate option is True, setImage is called with no angle,
// then the angle is calculated by Tesseract and then setImage is re-called.
// Otherwise, setImage is called once using the user-provided rotateRadiansFinal value.
let rotateRadiansFinal;
if (options.rotateAuto) {
// The angle is only detected if auto page segmentation is used
// Therefore, if this is not the mode specified by the user, it is enabled temporarily here
const psmInit = api.GetPageSegMode();
let psmEdit = false;
if (![PSM.AUTO, PSM.AUTO_ONLY, PSM.OSD].includes(String(psmInit))) {
psmEdit = true;
api.SetVariable('tessedit_pageseg_mode', String(PSM.AUTO));
}
setImage(TessModule, api, image);
api.FindLines();
// The function GetAngle will be replaced with GetGradient in 4.0.4,
// but for now we want to maintain compatibility.
// We can switch to only using GetGradient in v5.
const rotateRadiansCalc = api.GetGradient ? api.GetGradient() : api.GetAngle();
// Restore user-provided PSM setting
if (psmEdit) {
api.SetVariable('tessedit_pageseg_mode', String(psmInit));
}
// Small angles (<0.005 radians/~0.3 degrees) are ignored to save on runtime
if (Math.abs(rotateRadiansCalc) >= 0.005) {
rotateRadiansFinal = rotateRadiansCalc;
setImage(TessModule, api, image, rotateRadiansFinal);
} else {
// Image needs to be reset if run with different PSM setting earlier
if (psmEdit) {
setImage(TessModule, api, image);
}
rotateRadiansFinal = 0;
}
} else {
rotateRadiansFinal = options.rotateRadians || 0;
setImage(TessModule, api, image, rotateRadiansFinal);
}
const rec = options.rectangle;
if (typeof rec === 'object') {
api.SetRectangle(rec.left, rec.top, rec.width, rec.height);
}
if (!skipRecognition) {
api.Recognize(null);
} else {
if (output.layoutBlocks) {
api.AnalyseLayout();
}
log('Skipping recognition: all output options requiring recognition are disabled.');
}
const { pdfTitle } = options;
const { pdfTextOnly } = options;
const result = dump(TessModule, api, workingOutput, { pdfTitle, pdfTextOnly, skipRecognition });
result.rotateRadians = rotateRadiansFinal;
if (output.debug) TessModule.FS.unlink('/debugInternal.txt');
if (Object.keys(optionsTess).length > 0) {
api.RestoreParameters();
}
res.resolve(result);
} catch (err) {
res.reject(err.toString());
}
};
const detect = async ({ payload: { image } }, res) => {
try {
setImage(TessModule, api, image);
const results = new TessModule.OSResults();
if (!api.DetectOS(results)) {
res.resolve({
tesseract_script_id: null,
script: null,
script_confidence: null,
orientation_degrees: null,
orientation_confidence: null,
});
} else {
const best = results.best_result;
const oid = best.orientation_id;
const sid = best.script_id;
res.resolve({
tesseract_script_id: sid,
script: results.unicharset.get_script_from_script_id(sid),
script_confidence: best.sconfidence,
orientation_degrees: [0, 270, 180, 90][oid],
orientation_confidence: best.oconfidence,
});
}
} catch (err) {
res.reject(err.toString());
}
};
const terminate = async (_, res) => {
try {
if (api !== null) {
api.End();
}
res.resolve({ terminated: true });
} catch (err) {
res.reject(err.toString());
}
};
/**
* dispatchHandlers
*
* @name dispatchHandlers
* @function worker data handler
* @access public
* @param {object} data
* @param {string} data.jobId - unique job id
* @param {string} data.action - action of the job, only recognize and detect for now
* @param {object} data.payload - data for the job
* @param {function} send - trigger job to work
*/
exports.dispatchHandlers = (packet, send) => {
const res = (status, data) => {
// Return only the necessary info to avoid sending unnecessarily large messages
const packetRes = {
jobId: packet.jobId,
workerId: packet.workerId,
action: packet.action,
};
send({
...packetRes,
status,
data,
});
};
res.resolve = res.bind(this, 'resolve');
res.reject = res.bind(this, 'reject');
res.progress = res.bind(this, 'progress');
latestJob = res;
({
load,
FS,
loadLanguage,
initialize,
setParameters,
recognize,
detect,
terminate,
})[packet.action](packet, res)
.catch((err) => res.reject(err.toString()));
};
/**
* setAdapter
*
* @name setAdapter
* @function
* @access public
* @param {object} adapter - implementation of the worker, different in browser and node environment
*/
exports.setAdapter = (_adapter) => {
adapter = _adapter;
};
@@ -0,0 +1,18 @@
'use strict';
const util = require('util');
const fs = require('fs');
module.exports = {
readCache: util.promisify(fs.readFile),
writeCache: util.promisify(fs.writeFile),
deleteCache: (path) => (
util.promisify(fs.unlink)(path)
.catch(() => {})
),
checkCache: (path) => (
util.promisify(fs.access)(path, fs.F_OK)
.then((err) => (err === null))
.catch(() => false)
),
};
@@ -0,0 +1,38 @@
'use strict';
const { simd, relaxedSimd } = require('wasm-feature-detect');
const OEM = require('../../constants/OEM');
let TesseractCore = null;
/*
* getCore is a sync function to load and return
* TesseractCore.
*/
module.exports = async (oem, _, res) => {
if (TesseractCore === null) {
const statusText = 'loading tesseract core';
const simdSupport = await simd();
const relaxedSimdSupport = await relaxedSimd();
res.progress({ status: statusText, progress: 0 });
if (relaxedSimdSupport) {
if ([OEM.DEFAULT, OEM.LSTM_ONLY].includes(oem)) {
TesseractCore = require('tesseract.js-core/tesseract-core-relaxedsimd-lstm');
} else {
TesseractCore = require('tesseract.js-core/tesseract-core-relaxedsimd');
}
} else if (simdSupport) {
if ([OEM.DEFAULT, OEM.LSTM_ONLY].includes(oem)) {
TesseractCore = require('tesseract.js-core/tesseract-core-simd-lstm');
} else {
TesseractCore = require('tesseract.js-core/tesseract-core-simd');
}
} else if ([OEM.DEFAULT, OEM.LSTM_ONLY].includes(oem)) {
TesseractCore = require('tesseract.js-core/tesseract-core-lstm');
} else {
TesseractCore = require('tesseract.js-core/tesseract-core');
}
res.progress({ status: statusText, progress: 1 });
}
return TesseractCore;
};
@@ -0,0 +1,3 @@
'use strict';
module.exports = require('zlib').gunzipSync;
@@ -0,0 +1,32 @@
'use strict';
/**
*
* Tesseract Worker Script for Node
*
* @fileoverview Node worker implementation
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
// Use built-in fetch if available, otherwise fallback to node-fetch
const fetch = global.fetch || require('node-fetch');
const { parentPort } = require('worker_threads');
const worker = require('..');
const getCore = require('./getCore');
const gunzip = require('./gunzip');
const cache = require('./cache');
/*
* register message handler
*/
parentPort.on('message', (packet) => {
worker.dispatchHandlers(packet, (obj) => parentPort.postMessage(obj));
});
worker.setAdapter({
getCore,
gunzip,
fetch,
...cache,
});
@@ -0,0 +1,58 @@
'use strict';
// Copied from https://gist.github.com/jonleighton/958841
// Copyright 2011 Jon Leighton, MIT LICENSE
/* eslint no-bitwise: 0 */
module.exports = (arrayBuffer) => {
let base64 = '';
const encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
const bytes = new Uint8Array(arrayBuffer);
const { byteLength } = bytes;
const byteRemainder = byteLength % 3;
const mainLength = byteLength - byteRemainder;
let a; let b; let c; let
d;
let chunk;
// Main loop deals with bytes in chunks of 3
for (let i = 0; i < mainLength; i += 3) {
// Combine the three bytes into a single integer
chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
// Use bitmasks to extract 6-bit segments from the triplet
a = (chunk & 16515072) >> 18; // 16515072 = (2^6 - 1) << 18
b = (chunk & 258048) >> 12; // 258048 = (2^6 - 1) << 12
c = (chunk & 4032) >> 6; // 4032 = (2^6 - 1) << 6
d = chunk & 63; // 63 = 2^6 - 1
// Convert the raw binary segments to the appropriate ASCII encoding
base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d];
}
// Deal with the remaining bytes and padding
if (byteRemainder === 1) {
chunk = bytes[mainLength];
a = (chunk & 252) >> 2; // 252 = (2^6 - 1) << 2
// Set the 4 least significant bits to zero
b = (chunk & 3) << 4; // 3 = 2^2 - 1
base64 += `${encodings[a] + encodings[b]}==`;
} else if (byteRemainder === 2) {
chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1];
a = (chunk & 64512) >> 10; // 64512 = (2^6 - 1) << 10
b = (chunk & 1008) >> 4; // 1008 = (2^6 - 1) << 4
// Set the 2 least significant bits to zero
c = (chunk & 15) << 2; // 15 = 2^4 - 1
base64 += `${encodings[a] + encodings[b] + encodings[c]}=`;
}
return base64;
};
@@ -0,0 +1,89 @@
'use strict';
/**
*
* Dump data to a big JSON tree
*
* @fileoverview dump data to JSON tree
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
const arrayBufferToBase64 = require('./arrayBufferToBase64');
const imageType = require('../../constants/imageType');
/**
* deindent
*
* The generated HOCR is excessively indented, so
* we get rid of that indentation
*
* @name deindent
* @function deindent string
* @access public
*/
const deindent = (html) => {
const lines = html.split('\n');
if (lines[0].substring(0, 2) === ' ') {
for (let i = 0; i < lines.length; i += 1) {
if (lines[i].substring(0, 2) === ' ') {
lines[i] = lines[i].slice(2);
}
}
}
return lines.join('\n');
};
/**
* dump
*
* @name dump
* @function dump recognition result to a JSON object
* @access public
*/
module.exports = (TessModule, api, output, options) => {
const enumToString = (value, prefix) => (
Object.keys(TessModule)
.filter((e) => (e.startsWith(`${prefix}_`) && TessModule[e] === value))
.map((e) => e.slice(prefix.length + 1))[0]
);
const getImage = (type) => {
api.WriteImage(type, '/image.png');
const pngBuffer = TessModule.FS.readFile('/image.png');
const pngStr = `data:image/png;base64,${arrayBufferToBase64(pngBuffer.buffer)}`;
TessModule.FS.unlink('/image.png');
return pngStr;
};
const getPDFInternal = (title, textonly) => {
const pdfRenderer = new TessModule.TessPDFRenderer('tesseract-ocr', '/', textonly);
pdfRenderer.BeginDocument(title);
pdfRenderer.AddImage(api);
pdfRenderer.EndDocument();
TessModule._free(pdfRenderer);
return TessModule.FS.readFile('/tesseract-ocr.pdf');
};
return {
text: output.text ? api.GetUTF8Text() : null,
hocr: output.hocr ? deindent(api.GetHOCRText()) : null,
tsv: output.tsv ? api.GetTSVText() : null,
box: output.box ? api.GetBoxText() : null,
unlv: output.unlv ? api.GetUNLVText() : null,
osd: output.osd ? api.GetOsdText() : null,
pdf: output.pdf ? getPDFInternal(options.pdfTitle ?? 'Tesseract OCR Result', options.pdfTextOnly ?? false) : null,
imageColor: output.imageColor ? getImage(imageType.COLOR) : null,
imageGrey: output.imageGrey ? getImage(imageType.GREY) : null,
imageBinary: output.imageBinary ? getImage(imageType.BINARY) : null,
confidence: !options.skipRecognition ? api.MeanTextConf() : null,
blocks: output.blocks && !options.skipRecognition ? JSON.parse(api.GetJSONText()).blocks : null,
layoutBlocks: output.layoutBlocks && options.skipRecognition
? JSON.parse(api.GetJSONText()).blocks : null,
psm: enumToString(api.GetPageSegMode(), 'PSM'),
oem: enumToString(api.oem(), 'OEM'),
version: api.Version(),
debug: output.debug ? TessModule.FS.readFile('/debugInternal.txt', { encoding: 'utf8', flags: 'a+' }) : null,
};
};
@@ -0,0 +1,34 @@
'use strict';
const bmp = require('bmp-js');
/**
* setImage
*
* @name setImage
* @function set image in tesseract for recognition
* @access public
*/
module.exports = (TessModule, api, image, angle = 0) => {
// Check for bmp magic numbers (42 and 4D in hex)
const isBmp = (image[0] === 66 && image[1] === 77) || (image[1] === 66 && image[0] === 77);
const exif = parseInt(image.slice(0, 500).join(' ').match(/1 18 0 3 0 0 0 1 0 (\d)/)?.[1], 10) || 1;
// /*
// * Leptonica supports some but not all bmp files
// * @see https://github.com/DanBloomberg/leptonica/issues/607#issuecomment-1068802516
// * We therefore use bmp-js to convert all bmp files into a format Leptonica is known to support
// */
if (isBmp) {
// Not sure what this line actually does, but removing breaks the function
const buf = Buffer.from(Array.from({ ...image, length: Object.keys(image).length }));
const bmpBuf = bmp.decode(buf);
TessModule.FS.writeFile('/input', bmp.encode(bmpBuf).data);
} else {
TessModule.FS.writeFile('/input', image);
}
const res = api.SetImageFile(exif, angle);
if (res === 1) throw Error('Error attempting to read image.');
};
@@ -0,0 +1,12 @@
'use strict';
const version = require('../../../package.json').version;
const defaultOptions = require('../../constants/defaultOptions');
/*
* Default options for browser worker
*/
module.exports = {
...defaultOptions,
workerPath: `https://cdn.jsdelivr.net/npm/tesseract.js@v${version}/dist/worker.min.js`,
};
@@ -0,0 +1,26 @@
'use strict';
/**
*
* Tesseract Worker adapter for browser
*
* @fileoverview Tesseract Worker adapter for browser
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
const defaultOptions = require('./defaultOptions');
const spawnWorker = require('./spawnWorker');
const terminateWorker = require('./terminateWorker');
const onMessage = require('./onMessage');
const send = require('./send');
const loadImage = require('./loadImage');
module.exports = {
defaultOptions,
spawnWorker,
terminateWorker,
onMessage,
send,
loadImage,
};
@@ -0,0 +1,71 @@
'use strict';
/**
* readFromBlobOrFile
*
* @name readFromBlobOrFile
* @function
* @access private
*/
const readFromBlobOrFile = (blob) => (
new Promise((resolve, reject) => {
const fileReader = new FileReader();
fileReader.onload = () => {
resolve(fileReader.result);
};
fileReader.onerror = ({ target: { error: { code } } }) => {
reject(Error(`File could not be read! Code=${code}`));
};
fileReader.readAsArrayBuffer(blob);
})
);
/**
* loadImage
*
* @name loadImage
* @function load image from different source
* @access private
*/
const loadImage = async (image) => {
let data = image;
if (typeof image === 'undefined') {
return 'undefined';
}
if (typeof image === 'string') {
// Base64 Image
if (/data:image\/([a-zA-Z]*);base64,([^"]*)/.test(image)) {
data = atob(image.split(',')[1])
.split('')
.map((c) => c.charCodeAt(0));
} else {
const resp = await fetch(image);
data = await resp.arrayBuffer();
}
} else if (typeof HTMLElement !== 'undefined' && image instanceof HTMLElement) {
if (image.tagName === 'IMG') {
data = await loadImage(image.src);
}
if (image.tagName === 'VIDEO') {
data = await loadImage(image.poster);
}
if (image.tagName === 'CANVAS') {
await new Promise((resolve) => {
image.toBlob(async (blob) => {
data = await readFromBlobOrFile(blob);
resolve();
});
});
}
} else if (typeof OffscreenCanvas !== 'undefined' && image instanceof OffscreenCanvas) {
const blob = await image.convertToBlob();
data = await readFromBlobOrFile(blob);
} else if (image instanceof File || image instanceof Blob) {
data = await readFromBlobOrFile(image);
}
return new Uint8Array(data);
};
module.exports = loadImage;
@@ -0,0 +1,7 @@
'use strict';
module.exports = (worker, handler) => {
worker.onmessage = ({ data }) => { // eslint-disable-line
handler(data);
};
};
@@ -0,0 +1,12 @@
'use strict';
/**
* send
*
* @name send
* @function send packet to worker and create a job
* @access public
*/
module.exports = async (worker, packet) => {
worker.postMessage(packet);
};
@@ -0,0 +1,22 @@
'use strict';
/**
* spawnWorker
*
* @name spawnWorker
* @function create a new Worker in browser
* @access public
*/
module.exports = ({ workerPath, workerBlobURL }) => {
let worker;
if (Blob && URL && workerBlobURL) {
const blob = new Blob([`importScripts("${workerPath}");`], {
type: 'application/javascript',
});
worker = new Worker(URL.createObjectURL(blob));
} else {
worker = new Worker(workerPath);
}
return worker;
};
@@ -0,0 +1,12 @@
'use strict';
/**
* terminateWorker
*
* @name terminateWorker
* @function terminate worker
* @access public
*/
module.exports = (worker) => {
worker.terminate();
};
@@ -0,0 +1,12 @@
'use strict';
const path = require('path');
const defaultOptions = require('../../constants/defaultOptions');
/*
* Default options for node worker
*/
module.exports = {
...defaultOptions,
workerPath: path.join(__dirname, '..', '..', 'worker-script', 'node', 'index.js'),
};
@@ -0,0 +1,26 @@
'use strict';
/**
*
* Tesseract Worker impl. for node (using child_process)
*
* @fileoverview Tesseract Worker impl. for node
* @author Kevin Kwok <antimatter15@gmail.com>
* @author Guillermo Webster <gui@mit.edu>
* @author Jerome Wu <jeromewus@gmail.com>
*/
const defaultOptions = require('./defaultOptions');
const spawnWorker = require('./spawnWorker');
const terminateWorker = require('./terminateWorker');
const onMessage = require('./onMessage');
const send = require('./send');
const loadImage = require('./loadImage');
module.exports = {
defaultOptions,
spawnWorker,
terminateWorker,
onMessage,
send,
loadImage,
};
@@ -0,0 +1,38 @@
'use strict';
const util = require('util');
const fs = require('fs');
// Use built-in fetch if available, otherwise fallback to node-fetch
const fetch = global.fetch || require('node-fetch');
const isURL = require('is-url');
const readFile = util.promisify(fs.readFile);
/**
* loadImage
*
* @name loadImage
* @function load image from different source
* @access public
*/
module.exports = async (image) => {
let data = image;
if (typeof image === 'undefined') {
return image;
}
if (typeof image === 'string') {
if (isURL(image) || image.startsWith('moz-extension://') || image.startsWith('chrome-extension://') || image.startsWith('file://')) {
const resp = await fetch(image);
data = await resp.arrayBuffer();
} else if (/data:image\/([a-zA-Z]*);base64,([^"]*)/.test(image)) {
data = Buffer.from(image.split(',')[1], 'base64');
} else {
data = await readFile(image);
}
} else if (Buffer.isBuffer(image)) {
data = image;
}
return new Uint8Array(data);
};
@@ -0,0 +1,5 @@
'use strict';
module.exports = (worker, handler) => {
worker.on('message', handler);
};
@@ -0,0 +1,12 @@
'use strict';
/**
* send
*
* @name send
* @function send packet to worker and create a job
* @access public
*/
module.exports = async (worker, packet) => {
worker.postMessage(packet);
};
@@ -0,0 +1,12 @@
'use strict';
const { Worker } = require('worker_threads');
/**
* spawnWorker
*
* @name spawnWorker
* @function fork a new process in node
* @access public
*/
module.exports = ({ workerPath }) => new Worker(workerPath);
@@ -0,0 +1,12 @@
'use strict';
/**
* terminateWorker
*
* @name terminateWorker
* @function kill worker
* @access public
*/
module.exports = (worker) => {
worker.terminate();
};