From 04a4fa2953e2308456e098e4abf20e43fcbd6800 Mon Sep 17 00:00:00 2001 From: Kuba Orlik Date: Sun, 28 Mar 2021 08:35:17 +0200 Subject: [PATCH] Working aobrtable perspective search --- package-lock.json | 20 ++++ package.json | 4 +- src/abortable-promise.ts | 96 +++++++++++++++ src/alpr.ts | 252 +++++++++++++++++++++------------------ src/index.ts | 89 +++++++------- src/transforms.ts | 139 +++++++++++++++++++++ 6 files changed, 440 insertions(+), 160 deletions(-) create mode 100644 src/abortable-promise.ts create mode 100644 src/transforms.ts diff --git a/package-lock.json b/package-lock.json index 2c68a87..131f4d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "abortable-promise": "git+https://git.kuba-orlik.name/kuba/abortable-promises.git#ee3d998", "async-spawner": "git+https://git.kuba-orlik.name/kuba/async-spawner.git#7ee6a21", + "emittery": "^0.8.1", "tempy": "^1.0.1" }, "devDependencies": { @@ -53,6 +55,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.35.tgz", "integrity": "sha512-Lt+wj8NVPx0zUmUwumiVXapmaLUcAk3yPuHCFVXras9k5VT9TdhJqKqGVUQCD60OTMCl0qxJ57OiTL0Mic3Iag==" }, + "node_modules/abortable-promise": { + "version": "0.1.0", + "resolved": "git+https://git.kuba-orlik.name/kuba/abortable-promises.git#ee3d998c08b43a2efb15d921f592e8fdc433f487", + "hasInstallScript": true, + "license": "ISC", + "dependencies": { + "@types/node": "^14.14.19", + "emittery": "^0.8.1" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -606,6 +618,14 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.35.tgz", "integrity": "sha512-Lt+wj8NVPx0zUmUwumiVXapmaLUcAk3yPuHCFVXras9k5VT9TdhJqKqGVUQCD60OTMCl0qxJ57OiTL0Mic3Iag==" }, + "abortable-promise": { + "version": "git+https://git.kuba-orlik.name/kuba/abortable-promises.git#ee3d998c08b43a2efb15d921f592e8fdc433f487", + "from": "abortable-promise@git+https://git.kuba-orlik.name/kuba/abortable-promises.git#ee3d998", + "requires": { + "@types/node": "^14.14.19", + "emittery": "^0.8.1" + } + }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", diff --git a/package.json b/package.json index 25711a4..e744dc8 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ "author": "", "license": "ISC", "dependencies": { - "async-spawner": "git+https://git.kuba-orlik.name/kuba/async-spawner.git#7ee6a21", + "abortable-promise": "git+https://git.kuba-orlik.name/kuba/abortable-promises.git#4a316bb", + "async-spawner": "git+https://git.kuba-orlik.name/kuba/async-spawner.git#cdbb47d", + "emittery": "^0.8.1", "tempy": "^1.0.1" }, "devDependencies": { diff --git a/src/abortable-promise.ts b/src/abortable-promise.ts new file mode 100644 index 0000000..54829cd --- /dev/null +++ b/src/abortable-promise.ts @@ -0,0 +1,96 @@ +import { deferedSpawn } from "async-spawner"; +import Emittery from "emittery"; +import { promisify } from "util"; + +//needed to pass information on wether or not the promise has been aborted witout 'this' +class AbortableEmittery extends Emittery { + public aborted = false; + + abort() { + this.aborted = true; + this.emit("abort"); + } +} + +class AbortablePromise extends Promise { + private emitter: AbortableEmittery; + + constructor( + make_generator: ( + await_or_abort: ( + promise: Promise, + on_abort: () => void + ) => Promise + ) => AsyncGenerator + ) { + const emitter = new AbortableEmittery(); + const await_or_abort = ( + promise: Promise, + on_abort: () => void + ) => { + return new Promise((resolve, reject) => { + let resolved = false; + emitter.on("abort", () => { + if (!resolved) { + on_abort(); + resolve(null); + resolved = true; + } + }); + promise + .then((result) => { + if (!resolved) { + resolve(result); + resolved = true; + } + }) + .catch(reject); + }); + }; + super(async (resolve, reject) => { + let step_value; + const generator = make_generator(await_or_abort); + do { + if (emitter.aborted) { + resolve(null); + return; + } + step_value = await generator.next(); + } while (!step_value.done); + }); + this.emitter = emitter; + } + + abort() { + this.emitter.abort(); + } +} + +// const sleep = promisify((timeout: number, callback: (...args: any[]) => void) => +// setTimeout(callback, timeout) +// ); + +// const a = new AbortablePromise(async function* () { +// yield await sleep(1000); +// console.log("awaited 100"); +// yield await sleep(2000); +// console.log("awaited 200"); +// yield await sleep(3000); +// console.log("awaited 300"); +// }); + +// setTimeout(() => a.abort(), 1500); + +const b = new AbortablePromise(async function* (await_or_abort) { + const ping = await deferedSpawn("ping", ["8.8.8.8"]); + while (true) { + console.log( + await await_or_abort(ping.waitForNextData(), () => { + ping.kill(); + }) + ); + yield; + } +}); + +setTimeout(() => b.abort(), 5000); diff --git a/src/alpr.ts b/src/alpr.ts index 1f7e6d4..259f60c 100644 --- a/src/alpr.ts +++ b/src/alpr.ts @@ -1,24 +1,35 @@ -import simpleSpawn from "async-spawner"; +import AbortablePromise from "abortable-promise"; +import { deferedSpawn } from "async-spawner"; import tempy from "tempy"; +import { + newpos, + transformCoefs, + transformDesc, + transforms, +} from "./transforms"; -async function perspectiveShift( +function perspectiveShift( input: string, shift: transformCoefs -): Promise { - const output = tempy.file({ extension: "jpg" }); - const args = [ - "-verbose", - input, - "-virtual-pixel", - "Transparent", - "-distort", - "PerspectiveProjection", - shift.join(","), - output, - ]; - console.log(args); - await simpleSpawn("convert", args); - return output; // the file name; +): AbortablePromise { + return new AbortablePromise(async function* (await_or_abort) { + const output = tempy.file({ extension: "jpg" }); + const args = [ + "-verbose", + input, + "-virtual-pixel", + "Transparent", + "-distort", + "PerspectiveProjection", + shift.join(","), + output, + ]; + const def = await deferedSpawn("convert", args); + const next_data_promise = def.waitForNextData(); + yield; + await await_or_abort(next_data_promise, () => def.kill()); + return output; + }); } type coordinates = { x: number; y: number }; @@ -28,7 +39,7 @@ type alpr_candidate = { matches_template: 0 | 1; }; -type alpr_response = { +export type alpr_response = { version: number; data_type: "alpr_results"; epoch_time: number; @@ -68,113 +79,116 @@ function empty_alpr_response() { }; } -export default async function alpr(path: string): Promise { - const result = await simpleSpawn("alpr", [ - "-c", - "eu", - "-p", - "pl", - "-j", - path, - ]); - return JSON.parse(result.stdout); -} +type DeferedSpawn = ReturnType; -function newpos( - x: number, - y: number, - t: transformCoefs -): { x: number; y: number } { - return { - x: (x * t[0] + y * t[1] + t[2]) / (t[6] * x + t[7] * y + 1), - y: (x * t[3] + y * t[4] + t[5]) / (t[6] * x + t[7] * y + 1), - }; +export function alpr(path: string): AbortablePromise { + return new AbortablePromise(async function* (await_or_abort) { + console.log("Running alpr abortablepromise"); + const def = await deferedSpawn("alpr", [ + "-c", + "eu", + "-p", + "pl", + "-j", + path, + ]); + console.log("created deferedspawn"); + const answer_promise = def.waitForNextData(); + yield; + const answer = await await_or_abort(answer_promise, () => def.kill()); + console.log("got answer!"); + return JSON.parse((answer as unknown) as string); + }); } -export async function perspectiveAssistedAlpr( +export function perspectiveAssistedAlpr( input: string, shift: transformDesc -) { - const shifted_image = await perspectiveShift(input, shift.forward); - let result = await alpr(shifted_image); - for (const i in result.results) { - result.results[i].coordinates = result.results[i].coordinates.map( - (c) => { - // https://legacy.imagemagick.org/Usage/distorts/#perspective_internals - const ret = newpos(c.x, c.y, shift.backward); - console.log("new coords!", c, "→", ret); - return ret; - } - ) as [coordinates, coordinates, coordinates, coordinates]; - } - return result; -} - -type transformCoefs = [ - number, - number, - number, - number, - number, - number, - number, - number -]; -type transformDesc = { forward: transformCoefs; backward: transformCoefs }; - -// See perspective.md to learn how coefficients are created for calls below: -const transforms: transformDesc[] = [ - { - forward: [ - 0.418385, - -0.064912, - 1.11204e-12, - -1.07285e-16, - 0.407118, - 1.85136e-13, - -0.000143964, - -0.0000155811, - ], - backward: [ - 2.39014, - 0.381091, - -5.07071e-13, - 8.09745e-16, - 2.45629, - -1.95474e-12, - 0.000344095, - 0.0000931351, - ], - }, -]; - -export async function alprMultiPersp(input: string): Promise { - let resolved = false; - return new Promise((resolve, reject) => { - const callback = (arg: alpr_response) => { - if (resolved) { - return; - } - if (arg.results.length === 0) { - return; - } - resolved = true; - resolve(arg); - }; - Promise.all([ - alpr(input).then(callback), - ...transforms.map((transform) => - perspectiveAssistedAlpr(input, transform).then(callback) - ), - ]) - .then(() => { - if (!resolved) { - resolve(empty_alpr_response()); +): AbortablePromise { + console.log("new ap perspectiveAlpr"); + return new AbortablePromise(async function* (await_or_abort) { + const shifted_image = await await_or_abort( + perspectiveShift(input, shift.forward) + ); + yield; + console.log("after yield1"); + let result = await await_or_abort(alpr(shifted_image)); + yield; + console.log("after yield2"); + console.log("~~~Got perspective shifted response"); + if (result === null) { + return null; + } + for (const i in result.results) { + result.results[i].coordinates = result.results[i].coordinates.map( + (c) => { + // https://legacy.imagemagick.org/Usage/distorts/#perspective_internals + const ret = newpos(c.x, c.y, shift.backward); + console.log("new coords!", c, "→", ret); + return ret; } - }) - .catch(reject); + ) as [coordinates, coordinates, coordinates, coordinates]; + } + return result; }); } -console.log(newpos(4040, 0, transforms[0].forward)); -console.log(newpos(4040, 0, transforms[0].backward)); +// export function alprMultiPersp(input: string): AbortablePromise { +// return new AbortablePromise(async function* (await_or_abort) { +// let resolved = false; +// return new Promise((resolve, reject): void => { +// let promises: Array | Promise> = []; +// const callback = (arg: alpr_response) => { +// if (resolved) { +// return; +// } +// if (arg.results.length === 0) { +// return; +// } +// resolved = true; +// for (const promise of promises) { +// if (promise instanceof AbortablePromise) { +// console.log("aovrting promise!"); +// promise.abort(); +// } +// } +// resolve(arg); +// }; +// promises = [ +// alpr(input).then(callback), +// ...transforms.map((transform) => +// perspectiveAssistedAlpr(input, transform).then(callback) +// ), +// ]; +// Promise.all(promises) +// .then(() => { +// if (!resolved) { +// resolve(empty_alpr_response()); +// } +// }) +// .catch(reject); +// }); +// }); +// } +// +export function alprMultiPersp(input: string): AbortablePromise { + const ret = AbortablePromise.deadlyRace( + [ + alpr(input), + // ...transforms.map((transform) => + // perspectiveAssistedAlpr(input, transform) + // ), + perspectiveAssistedAlpr(input, transforms[0]), + ], + async (arg: alpr_response) => { + return arg.results.length > 0; + } + ); + return ret; + // return new Promise((resolve, reject) => { + // alpr(input).then(resolve, reject); + // }); + // return new Promise((resolve, reject) => { + // perspectiveAssistedAlpr(input, transforms[0]); + // }); +} diff --git a/src/index.ts b/src/index.ts index 75ce878..8c73988 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,55 +1,64 @@ import simpleSpawn, { deferedSpawn } from "async-spawner"; import { promises as fs } from "fs"; import path from "path"; -import { alprMultiPersp } from "./alpr"; +import AbortablePromise from "../../abortable-promise/@types"; +import { alprMultiPersp, alpr_response } from "./alpr"; async function run(): Promise { const gui = await deferedSpawn("python", [ "/home/kuba/projects/personal/qml-player/main.py", "/home/kuba/projects/personal/qml-player/template.qml", ]); - const dir = - "/home/kuba/projects/personal/zgłaszańska/03-20_2/problematic_cars"; + const dir = "/home/kuba/projects/personal/zgłaszańska/03-20_2/"; const files = await fs.readdir(dir); + let multi_perp_promise: AbortablePromise | null = null; for (const file of files.filter((f) => f.includes("jpg"))) { const full_path = path.resolve(dir, file); - void alprMultiPersp(full_path).then((o) => { - console.log("Got coords!"); - const { img_width, img_height } = o; - if (o.results.length) { - const min_x = Math.min( - ...o.results[0].coordinates.map((c: any) => c.x) - ); - const max_x = Math.max( - ...o.results[0].coordinates.map((c: any) => c.x) - ); - const min_y = Math.min( - ...o.results[0].coordinates.map((c: any) => c.y) - ); - const max_y = Math.max( - ...o.results[0].coordinates.map((c: any) => c.y) - ); + if (multi_perp_promise) { + multi_perp_promise.abort(); + } + multi_perp_promise = alprMultiPersp(full_path); + multi_perp_promise + .then((o) => { + if (o === null) return; + console.log("Got coords!"); + const { img_width, img_height } = o; + if (o.results.length) { + console.log(o.results[0]); + const min_x = Math.min( + ...o.results[0].coordinates.map((c: any) => c.x) + ); + const max_x = Math.max( + ...o.results[0].coordinates.map((c: any) => c.x) + ); + const min_y = Math.min( + ...o.results[0].coordinates.map((c: any) => c.y) + ); + const max_y = Math.max( + ...o.results[0].coordinates.map((c: any) => c.y) + ); - // console.log({ - // rect_x: min_x / img_width, - // rect_y: min_y / img_height, - // rect_width: (max_x - min_x) / img_width, - // rect_height: (max_y - min_y) / img_height, - // rect_text: o.results[0].candidates[0].plate, - // }); - gui.write( - JSON.stringify({ - rect_x: min_x / img_width, - rect_y: min_y / img_height, - rect_width: (max_x - min_x) / img_width, - rect_height: (max_y - min_y) / img_height, - rect_text: o.results[0].candidates[0].plate, - }) - ); - } else { - gui.write(JSON.stringify({ rect_width: 0 })); - } - }); + // console.log({ + // rect_x: min_x / img_width, + // rect_y: min_y / img_height, + // rect_width: (max_x - min_x) / img_width, + // rect_height: (max_y - min_y) / img_height, + // rect_text: o.results[0].candidates[0].plate, + // }); + gui.write( + JSON.stringify({ + rect_x: min_x / img_width, + rect_y: min_y / img_height, + rect_width: (max_x - min_x) / img_width, + rect_height: (max_y - min_y) / img_height, + rect_text: o.results[0].candidates[0].plate, + }) + ); + } else { + gui.write(JSON.stringify({ rect_width: 0 })); + } + }) + .catch(console.error); gui.write( JSON.stringify({ current_image: full_path, @@ -64,4 +73,4 @@ async function run(): Promise { } } -run(); +run().catch(console.error); diff --git a/src/transforms.ts b/src/transforms.ts new file mode 100644 index 0000000..dbf689b --- /dev/null +++ b/src/transforms.ts @@ -0,0 +1,139 @@ +export function newpos( + x: number, + y: number, + t: transformCoefs +): { x: number; y: number } { + return { + x: (x * t[0] + y * t[1] + t[2]) / (t[6] * x + t[7] * y + 1), + y: (x * t[3] + y * t[4] + t[5]) / (t[6] * x + t[7] * y + 1), + }; +} + +export type transformCoefs = [ + number, + number, + number, + number, + number, + number, + number, + number +]; +export type transformDesc = { + forward: transformCoefs; + backward: transformCoefs; +}; + +// See perspective.md to learn how coefficients are created for calls below: +export const transforms: transformDesc[] = [ + { + forward: [ + 0.418385, + -0.064912, + 1.11204e-12, + -1.07285e-16, + 0.407118, + 1.85136e-13, + -0.000143964, + -0.0000155811, + ], + backward: [ + 2.39014, + 0.381091, + -5.07071e-13, + 8.09745e-16, + 2.45629, + -1.95474e-12, + 0.000344095, + 0.0000931351, + ], + }, + { + forward: [ + 1.52043, + 3.31591e-16, + -9.50693e-13, + 4.91024e-16, + 1.29283, + -4.86523e-28, + 0.00012882, + 0.000072315, + ], + backward: [ + 0.657707, + 2.87435e-16, + -6.72954e-13, + 7.80686e-16, + 0.773495, + -1.67079e-12, + -0.0000847259, + -0.0000559353, + ], + }, + { + forward: [ + 0.894876, + -0.285098, + 8.16135e-13, + -4.48986e-17, + 1.06368, + 1.8139e-13, + -0.0000260208, + -0.0000110713, + ], + backward: [ + 1.11747, + 0.299515, + -4.53809e-13, + 4.55732e-16, + 0.940129, + -4.27521e-13, + 0.0000290775, + 0.0000182021, + ], + }, + { + forward: [ + 1.96875, + 8.8124e-16, + -1.34293e-12, + 0.725124, + 1, + -8.35683e-13, + 0.00023979, + 7.05268e-19, + ], + backward: [ + 0.507937, + -2.73557e-16, + 4.90838e-13, + -0.368317, + 1, + -1.03804e-12, + -0.000121798, + -1.49027e-19, + ], + }, + { + forward: [ + 3.88517, + 0.106038, + -3.10281e-12, + 1.43097, + 0.722206, + -5.02081e-13, + 0.00071415, + -0.0000918632, + ], + backward: [ + 0.272104, + -0.0399518, + -6.63745e-13, + -0.539145, + 1.46381, + -8.91691e-12, + -0.000243851, + 0.000163002, + ], + }, +];