diff --git a/Apps/UnitTests/Assets/image-reload.png b/Apps/UnitTests/Assets/image-reload.png new file mode 100644 index 000000000..206666ad9 Binary files /dev/null and b/Apps/UnitTests/Assets/image-reload.png differ diff --git a/Apps/UnitTests/CMakeLists.txt b/Apps/UnitTests/CMakeLists.txt index 3371afc47..e0d75044e 100644 --- a/Apps/UnitTests/CMakeLists.txt +++ b/Apps/UnitTests/CMakeLists.txt @@ -20,6 +20,7 @@ set(TEST_ASSETS "JavaScript/dist/tests.shaderCompilation.comprehensiveGLSL.js") set(PACKAGED_TEST_ASSETS + "Assets/image-reload.png" "Assets/droidsans.ttf" "Assets/droidsans.LICENSE.txt") @@ -88,6 +89,7 @@ endif() add_executable(UnitTests ${BABYLONJS_ASSETS} ${BABYLONJS_MATERIALS_ASSETS} ${TEST_ASSETS} ${PACKAGED_TEST_ASSETS} ${TEST_SCRIPTS} ${SOURCES}) if(BABYLON_NATIVE_PLUGIN_NATIVEENGINE_LOAD_IMAGES) target_compile_definitions(UnitTests PRIVATE HAS_NATIVE_IMAGE_LOADING) + target_link_libraries(UnitTests PRIVATE UrlLib) endif() target_include_directories(UnitTests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../Plugins/NativeEngine/Source") target_include_directories(UnitTests PRIVATE "${BGFX_DIR}/3rdparty") diff --git a/Apps/UnitTests/JavaScript/dist/tests.javaScript.all.js b/Apps/UnitTests/JavaScript/dist/tests.javaScript.all.js index 58440130c..d790cbd1a 100644 --- a/Apps/UnitTests/JavaScript/dist/tests.javaScript.all.js +++ b/Apps/UnitTests/JavaScript/dist/tests.javaScript.all.js @@ -30618,6 +30618,148 @@ function hexToBytes(hex) { }); }); +describe("Canvas image reloads", function () { + this.timeout(5000); + var test = hasNativeImageLoading ? it : it.skip; + var pixelTest = hasNativeImageLoading && hasGpuRendering && !skipCanvasGpuTests ? it : it.skip; + var url = "app:///Assets/image-reload.png"; + var dataUrl = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAMAAAABCAAAAAA+i0toAAAADElEQVR42mNgqP8PAAIBAX+LG2RhAAAAAElFTkSuQmCC"; + var redPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR42mP4z8DwH4QBEfcD/f6tu5kAAAAASUVORK5CYII="; + var greenPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADElEQVR42mNg+A+BAA/5A/3mxKLtAAAAAElFTkSuQmCC"; + var expect = chai__WEBPACK_IMPORTED_MODULE_4__.expect; + + [[url, url], [url, dataUrl], [dataUrl, url], [dataUrl, dataUrl]].forEach(function (sources) { + test("loads the same image again (" + sources.map(function (source) { + return source === url ? "URL" : "data"; + }).join(" to ") + ")", function () { + var image = new _native.Image(); + return sources.reduce(function (pending, source) { + return pending.then(function () { + return new Promise(function (resolve, reject) { + image.onload = resolve; + image.onerror = reject; + image.src = source; + }).then(function () { + expect(image.width).to.equal(source === url ? 2 : 3); + expect(image.naturalWidth).to.equal(image.width); + expect(image.height).to.equal(1); + expect(image.naturalHeight).to.equal(1); + }); + }); + }, Promise.resolve()); + }); + }); + + test("reflects the assigned src immediately", function () { + var image = new _native.Image(); + return new Promise(function (resolve, reject) { + image.onload = resolve; + image.onerror = reject; + image.src = dataUrl; + expect(image.src).to.equal(dataUrl); + }).then(function () { + expect(image.src).to.equal(dataUrl); + }); + }); + + [true, false].forEach(function (fromUrl) { + test("only delivers the latest assignment after a pending " + (fromUrl ? "URL" : "data") + " load", function () { + setImageReloadTestResponse(buffer__WEBPACK_IMPORTED_MODULE_5__.Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR4nGP4z8AAQv8BD/kD/YURmXYAAAAASUVORK5CYII=", "base64")); + var image = new _native.Image(); + var barrier = new _native.Image(); + var loaded = []; + var errors = []; + return new Promise(function (resolve, reject) { + image.onload = function () { loaded.push(image.width); }; + image.onerror = function (error) { errors.push(error); reject(error); }; + barrier.onload = resolve; + barrier.onerror = reject; + image.src = fromUrl ? "image-reload-test:///image.png" : dataUrl; + image.src = dataUrl; + barrier.src = dataUrl; + }).then(function () { + expect(image.src).to.equal(dataUrl); + expect(loaded).to.deep.equal([3]); + expect(errors).to.deep.equal([]); + expect(image.width).to.equal(3); + expect(barrier.width).to.equal(3); + }); + }); + }); + + [false, true].forEach(function (toData) { + pixelTest("draws reloaded pixels instead of the cached texture (URL to " + (toData ? "data" : "URL") + ")", function () { + var canvas = new _native.Canvas(); + canvas.width = 4; + canvas.height = 4; + var context = canvas.getContext("2d"); + var image = new _native.Image(); + function load(assign) { + return new Promise(function (resolve, reject) { + image.onload = resolve; + image.onerror = reject; + assign(); + }); + } + function expectChannel(channel) { + var data = context.getImageData(1, 1, 1, 1).data; + expect(data[channel]).to.be.greaterThan(240); + expect(data[1 - channel]).to.be.lessThan(20); + expect(data[2]).to.be.lessThan(20); + expect(data[3]).to.be.greaterThan(240); + } + return load(function () { + setImageReloadTestResponse(buffer__WEBPACK_IMPORTED_MODULE_5__.Buffer.from(redPng, "base64")); + image.src = "image-reload-test:///red.png"; + }).then(function () { + context.drawImage(image, 0, 0, 4, 4); + expectChannel(0); + return load(function () { + if (toData) { + image.src = "data:image/png;base64," + greenPng; + } else { + setImageReloadTestResponse(buffer__WEBPACK_IMPORTED_MODULE_5__.Buffer.from(greenPng, "base64")); + image.src = "image-reload-test:///green.png"; + } + }); + }).then(function () { + context.clearRect(0, 0, 4, 4); + context.drawImage(image, 0, 0, 4, 4); + expectChannel(1); + context.dispose(); + canvas.dispose(); + }, function (error) { + context.dispose(); + canvas.dispose(); + throw error; + }); + }); + }); + + test("reports load errors and can recover with data and URL loads", function () { + var image = new _native.Image(); + var errorCount = 0; + return new Promise(function (resolve, reject) { + image.onload = function () { reject(new Error("A missing file unexpectedly loaded")); }; + image.onerror = function () { ++errorCount; resolve(); }; + image.src = "app:///Assets/nonexistent-image-reload.png"; + }).then(function () { + expect(errorCount).to.equal(1); + return [dataUrl, url].reduce(function (pending, source) { + return pending.then(function () { + return new Promise(function (resolve, reject) { + image.onload = resolve; + image.onerror = reject; + image.src = source; + }).then(function () { + expect(image.src).to.equal(source); + expect(image.width).to.equal(source === url ? 2 : 3); + }); + }); + }, Promise.resolve()); + }); + }); +}); mocha.run(function (failures) { // Test program will wait for code to be set before exiting if (failures > 0) { diff --git a/Apps/UnitTests/JavaScript/src/tests.javaScript.all.ts b/Apps/UnitTests/JavaScript/src/tests.javaScript.all.ts index 0e9ef4636..d861bf035 100644 --- a/Apps/UnitTests/JavaScript/src/tests.javaScript.all.ts +++ b/Apps/UnitTests/JavaScript/src/tests.javaScript.all.ts @@ -36,6 +36,7 @@ declare const hostPlatform: string; declare const hasGpuRendering: boolean; declare const hasNativeImageLoading: boolean; declare const setExitCode: (code: number) => void; +declare const setImageReloadTestResponse: (bytes: Uint8Array) => void; declare const skipCanvasGpuTests: boolean; declare const _native: any; @@ -2125,6 +2126,138 @@ function hexToBytes(hex: string): Uint8Array { }); }); +describe("Canvas image reloads", function () { + this.timeout(5000); + const test = hasNativeImageLoading ? it : it.skip; + const url = "app:///Assets/image-reload.png"; + const dataUrl = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAMAAAABCAAAAAA+i0toAAAADElEQVR42mNgqP8PAAIBAX+LG2RhAAAAAElFTkSuQmCC"; + + for (const sources of [[url, url], [url, dataUrl], [dataUrl, url], [dataUrl, dataUrl]]) { + test(`loads the same image again (${sources.map(source => source === url ? "URL" : "data").join(" to ")})`, async function () { + const image = new _native.Image(); + for (const source of sources) { + await new Promise((resolve, reject) => { + image.onload = resolve; + image.onerror = reject; + image.src = source; + }); + expect(image.width).to.equal(source === url ? 2 : 3); + expect(image.naturalWidth).to.equal(image.width); + expect(image.height).to.equal(1); + expect(image.naturalHeight).to.equal(1); + } + }); + } + + test("reflects the assigned src immediately", async function () { + const image = new _native.Image(); + await new Promise((resolve, reject) => { + image.onload = resolve; + image.onerror = reject; + image.src = dataUrl; + expect(image.src).to.equal(dataUrl); + }); + expect(image.src).to.equal(dataUrl); + }); + + for (const fromUrl of [true, false]) { + test(`only delivers the latest assignment after a pending ${fromUrl ? "URL" : "data"} load`, async function () { + setImageReloadTestResponse(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR4nGP4z8AAQv8BD/kD/YURmXYAAAAASUVORK5CYII=", "base64")); + const image = new _native.Image(); + const barrier = new _native.Image(); + const loaded: number[] = []; + const errors: unknown[] = []; + await new Promise((resolve, reject) => { + image.onload = () => { loaded.push(image.width); }; + image.onerror = (error: unknown) => { errors.push(error); reject(error); }; + barrier.onload = resolve; + barrier.onerror = reject; + // The in-memory URL resolver queues completion synchronously. The final data load + // is a runtime-queue barrier after both candidate callbacks, not a timing estimate. + image.src = fromUrl ? "image-reload-test:///image.png" : dataUrl; + image.src = dataUrl; + barrier.src = dataUrl; + }); + expect(image.src).to.equal(dataUrl); + expect(loaded).to.deep.equal([3]); + expect(errors).to.deep.equal([]); + expect(image.width).to.equal(3); + expect(barrier.width).to.equal(3); + }); + } + + const pixelTest = hasNativeImageLoading && hasGpuRendering && !skipCanvasGpuTests ? it : it.skip; + const redPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR42mP4z8DwH4QBEfcD/f6tu5kAAAAASUVORK5CYII="; + const greenPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADElEQVR42mNg+A+BAA/5A/3mxKLtAAAAAElFTkSuQmCC"; + for (const toData of [false, true]) { + pixelTest(`draws reloaded pixels instead of the cached texture (URL to ${toData ? "data" : "URL"})`, async function () { + const canvas = new _native.Canvas(); + canvas.width = 4; + canvas.height = 4; + const context = canvas.getContext("2d"); + try { + const image = new _native.Image(); + const load = (assign: () => void) => new Promise((resolve, reject) => { + image.onload = resolve; + image.onerror = reject; + assign(); + }); + const sample = () => context.getImageData(1, 1, 1, 1).data; + const expectChannel = (data: any, channel: number) => { + expect(data[channel]).to.be.greaterThan(240); + expect(data[1 - channel]).to.be.lessThan(20); + expect(data[2]).to.be.lessThan(20); + expect(data[3]).to.be.greaterThan(240); + }; + + setImageReloadTestResponse(Buffer.from(redPng, "base64")); + await load(() => { + image.src = "image-reload-test:///red.png"; + }); + context.drawImage(image, 0, 0, 4, 4); + expectChannel(sample(), 0); + + if (toData) { + await load(() => { + image.src = "data:image/png;base64," + greenPng; + }); + } else { + setImageReloadTestResponse(Buffer.from(greenPng, "base64")); + await load(() => { + image.src = "image-reload-test:///green.png"; + }); + } + context.clearRect(0, 0, 4, 4); + context.drawImage(image, 0, 0, 4, 4); + expectChannel(sample(), 1); + } finally { + context.dispose(); + canvas.dispose(); + } + }); + } + + test("reports load errors and can recover with data and URL loads", async function () { + const image = new _native.Image(); + let errorCount = 0; + await new Promise((resolve, reject) => { + image.onload = () => reject(new Error("A missing file unexpectedly loaded")); + image.onerror = () => { ++errorCount; resolve(); }; + image.src = "app:///Assets/nonexistent-image-reload.png"; + }); + expect(errorCount).to.equal(1); + for (const source of [dataUrl, url]) { + await new Promise((resolve, reject) => { + image.onload = resolve; + image.onerror = reject; + image.src = source; + }); + expect(image.src).to.equal(source); + expect(image.width).to.equal(source === url ? 2 : 3); + } + }); +}); + mocha.run((failures) => { // Test program will wait for code to be set before exiting if (failures > 0) { diff --git a/Apps/UnitTests/Source/Tests.JavaScript.cpp b/Apps/UnitTests/Source/Tests.JavaScript.cpp index c396048d9..2c48f6d36 100644 --- a/Apps/UnitTests/Source/Tests.JavaScript.cpp +++ b/Apps/UnitTests/Source/Tests.JavaScript.cpp @@ -16,9 +16,14 @@ #include #endif #include +#ifdef HAS_NATIVE_IMAGE_LOADING +#include +#include +#include +#include +#endif #include -#include extern Babylon::Graphics::Configuration g_deviceConfig; @@ -107,6 +112,12 @@ TEST(JavaScript, All) // Change this to true to wait for the JavaScript debugger to attach (only applies to V8) constexpr const bool waitForDebugger = false; +#ifdef HAS_NATIVE_IMAGE_LOADING + const auto unregisterImageTestScheme = gsl::finally([] { + UrlLib::UrlRequest::UnregisterSchemeResolver("image-reload-test"); + }); +#endif + Babylon::Graphics::Device device{g_deviceConfig}; // Start rendering a frame to unblock the JavaScript from queuing graphics @@ -176,6 +187,21 @@ TEST(JavaScript, All) }, "setExitCode"); env.Global().Set("setExitCode", setExitCodeCallback); +#ifdef HAS_NATIVE_IMAGE_LOADING + env.Global().Set("setImageReloadTestResponse", Napi::Function::New(env, [](const Napi::CallbackInfo& info) { + const auto bytes = info[0].As(); + const auto begin = reinterpret_cast(bytes.Data()); + auto body = std::make_shared>(begin, begin + bytes.ElementLength()); + UrlLib::UrlRequest::RegisterSchemeResolver("image-reload-test", [body](const std::string&) { + UrlLib::UrlSchemeResolverResult result; + result.handled = true; + result.statusCode = UrlLib::UrlStatusCode::Ok; + result.contentType = "image/png"; + result.body = body; + return result; + }); + })); +#endif }); Babylon::ScriptLoader loader{runtime}; diff --git a/Polyfills/Canvas/Source/Context.cpp b/Polyfills/Canvas/Source/Context.cpp index 898b4f01f..b26d762f5 100644 --- a/Polyfills/Canvas/Source/Context.cpp +++ b/Polyfills/Canvas/Source/Context.cpp @@ -181,7 +181,7 @@ namespace Babylon::Polyfills::Internal ReleaseImagesAfterFlush(); for (auto& image : m_nvgImageIndices) { - nvgDeleteImage(*m_nvg, image.second); + nvgDeleteImage(*m_nvg, image.second.index); } nvgDelete(*m_nvg); m_nvg = nullptr; @@ -1394,15 +1394,24 @@ namespace Babylon::Polyfills::Internal } int imageIndex{-1}; + const uint32_t generation = canvasImage->GetContentGeneration(); const auto nvgImageIter = m_nvgImageIndices.find(canvasImage); - if (nvgImageIter == m_nvgImageIndices.end()) + // SetBuffer keeps this NativeCanvasImage* and only replaces its pixels. A same-size + // reload would otherwise keep drawing the texture created from the old buffer. + // Hold the old handle until this frame flushes so an earlier draw can still sample it. + if (nvgImageIter == m_nvgImageIndices.end() || nvgImageIter->second.generation != generation) { + if (nvgImageIter != m_nvgImageIndices.end()) + { + RetainImageUntilFlush(nvgImageIter->second.index); + m_nvgImageIndices.erase(nvgImageIter); + } imageIndex = canvasImage->CreateNVGImageForContext(*m_nvg); - m_nvgImageIndices.try_emplace(canvasImage, imageIndex); + m_nvgImageIndices.try_emplace(canvasImage, CachedNVGImage{imageIndex, generation}); } else { - imageIndex = nvgImageIter->second; + imageIndex = nvgImageIter->second.index; } assert(imageIndex != -1); diff --git a/Polyfills/Canvas/Source/Context.h b/Polyfills/Canvas/Source/Context.h index 97655947c..9c6afacc7 100644 --- a/Polyfills/Canvas/Source/Context.h +++ b/Polyfills/Canvas/Source/Context.h @@ -199,7 +199,12 @@ namespace Babylon::Polyfills::Internal std::shared_ptr m_cancellationSource{}; JsRuntimeScheduler m_runtimeScheduler; - std::unordered_map m_nvgImageIndices; + struct CachedNVGImage + { + int index{-1}; + uint32_t generation{0}; + }; + std::unordered_map m_nvgImageIndices; // Transient ImageData/canvas snapshots stay alive until nvgEndFrame has // consumed every queued draw that references them. std::vector m_imagesPendingFlush; diff --git a/Polyfills/Canvas/Source/Image.cpp b/Polyfills/Canvas/Source/Image.cpp index d21f869bd..306a5b781 100644 --- a/Polyfills/Canvas/Source/Image.cpp +++ b/Polyfills/Canvas/Source/Image.cpp @@ -56,6 +56,12 @@ namespace Babylon::Polyfills::Internal } void NativeCanvasImage::Dispose() + { + ReleaseImage(); + m_cancellationSource->cancel(); + } + + void NativeCanvasImage::ReleaseImage() { #ifdef BABYLON_NATIVE_PLUGIN_NATIVEENGINE_LOAD_IMAGES if (m_imageContainer) @@ -64,7 +70,6 @@ namespace Babylon::Polyfills::Internal m_imageContainer = nullptr; } #endif - m_cancellationSource->cancel(); } Napi::Value NativeCanvasImage::GetWidth(const Napi::CallbackInfo&) @@ -106,6 +111,7 @@ namespace Babylon::Polyfills::Internal bool NativeCanvasImage::SetBuffer(gsl::span buffer) { #ifdef BABYLON_NATIVE_PLUGIN_NATIVEENGINE_LOAD_IMAGES + ReleaseImage(); auto& allocator = Graphics::DeviceContext::GetDefaultAllocator(); m_imageContainer = bimg::imageParse(&allocator, buffer.data(), static_cast(buffer.size_bytes())); if (m_imageContainer != nullptr) @@ -126,6 +132,8 @@ namespace Babylon::Polyfills::Internal m_width = m_imageContainer->m_width; m_height = m_imageContainer->m_height; + // Bump before onload. A draw in that callback must not reuse the previous texture. + ++m_contentGeneration; if (!m_onloadHandlerRef.IsEmpty()) { @@ -146,13 +154,22 @@ namespace Babylon::Polyfills::Internal return; #else auto text{value.As().Utf8Value()}; + m_src = text; + m_cancellationSource->cancel(); + m_cancellationSource = std::make_shared(); // try with base64 static const std::string base64{"base64,"}; const auto pos = text.find(base64); if (pos != std::string::npos) { - arcana::make_task(m_runtimeScheduler, *m_cancellationSource, [env{info.Env()}, this, text{std::move(text)}, pos]() { + // SetSrc, disposal, decoding and event delivery share the JS runtime thread; + // cancellation cannot interleave with this synchronous decode. + arcana::make_task(m_runtimeScheduler, *m_cancellationSource, [env{info.Env()}, this, cancellationSource{m_cancellationSource}, text{std::move(text)}, pos]() { + if (cancellationSource->cancelled()) + { + return; + } std::vector base64Buffer; bn::decode_b64(text.begin() + pos + base64.length(), text.end(), std::back_inserter(base64Buffer)); gsl::span buffer = {reinterpret_cast(base64Buffer.data()), base64Buffer.size()}; @@ -169,15 +186,17 @@ namespace Babylon::Polyfills::Internal UrlLib::UrlRequest request{}; request.Open(UrlLib::UrlMethod::Get, text); request.ResponseType(UrlLib::UrlResponseType::Buffer); - request.SendAsync().then(m_runtimeScheduler, *m_cancellationSource, [env{info.Env()}, this, cancellationSource{m_cancellationSource}, request{std::move(request)}, text](arcana::expected result) { + request.SendAsync().then(m_runtimeScheduler, *m_cancellationSource, [env{info.Env()}, this, cancellationSource{m_cancellationSource}, request{std::move(request)}](arcana::expected result) { + if (cancellationSource->cancelled()) + { + return; + } if (result.has_error()) { HandleLoadImageError(Napi::Error::New(env, result.error())); return; } - Dispose(); - auto buffer{request.ResponseBuffer()}; if (buffer.data() == nullptr || buffer.size_bytes() == 0) { diff --git a/Polyfills/Canvas/Source/Image.h b/Polyfills/Canvas/Source/Image.h index 1d5b5db09..bc40b6025 100644 --- a/Polyfills/Canvas/Source/Image.h +++ b/Polyfills/Canvas/Source/Image.h @@ -26,6 +26,9 @@ namespace Babylon::Polyfills::Internal uint32_t GetWidth() const { return m_width; } uint32_t GetHeight() const { return m_height; } + // Contexts cache a NanoVG image by this pointer. This changes whenever SetBuffer + // replaces the decoded pixels, including a same-size URL or data reload. + uint32_t GetContentGeneration() const { return m_contentGeneration; } private: Napi::Value GetWidth(const Napi::CallbackInfo&); @@ -39,10 +42,12 @@ namespace Babylon::Polyfills::Internal void SetOnerror(const Napi::CallbackInfo&, const Napi::Value&); void HandleLoadImageError(const Napi::Error& error); bool SetBuffer(gsl::span buffer); + void ReleaseImage(); void Dispose(); uint32_t m_width{1}; uint32_t m_height{1}; + uint32_t m_contentGeneration{0}; std::string m_src{};