Fix Canvas image reload and cancellation lifetimes

Apps/UnitTests/CMakeLists.txt

@@ -20,6 +20,7 @@ set(TEST_ASSETS
     "JavaScript/dist/tests.shaderCompilation.comprehensiveGLSL.js")
 
 set(PACKAGED_TEST_ASSETS
+    "Assets/image-reload.png"
     "Assets/droidsans.ttf"
     "Assets/droidsans.LICENSE.txt")
 
@@ -88,6 +89,7 @@ endif()
 add_executable(UnitTests ${BABYLONJS_ASSETS} ${BABYLONJS_MATERIALS_ASSETS} ${TEST_ASSETS} ${PACKAGED_TEST_ASSETS} ${TEST_SCRIPTS} ${SOURCES})
 if(BABYLON_NATIVE_PLUGIN_NATIVEENGINE_LOAD_IMAGES)
     target_compile_definitions(UnitTests PRIVATE HAS_NATIVE_IMAGE_LOADING)
+    target_link_libraries(UnitTests PRIVATE UrlLib)
 endif()
 target_include_directories(UnitTests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../Plugins/NativeEngine/Source")
 target_include_directories(UnitTests PRIVATE "${BGFX_DIR}/3rdparty")

Apps/UnitTests/JavaScript/dist/tests.javaScript.all.js

@@ -30618,6 +30618,148 @@ function hexToBytes(hex) {
   });
 });
 
+describe("Canvas image reloads", function () {
+  this.timeout(5000);
+  var test = hasNativeImageLoading ? it : it.skip;
+  var pixelTest = hasNativeImageLoading && hasGpuRendering && !skipCanvasGpuTests ? it : it.skip;
+  var url = "app:///Assets/image-reload.png";
+  var dataUrl = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAMAAAABCAAAAAA+i0toAAAADElEQVR42mNgqP8PAAIBAX+LG2RhAAAAAElFTkSuQmCC";
+  var redPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR42mP4z8DwH4QBEfcD/f6tu5kAAAAASUVORK5CYII=";
+  var greenPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADElEQVR42mNg+A+BAA/5A/3mxKLtAAAAAElFTkSuQmCC";
+  var expect = chai__WEBPACK_IMPORTED_MODULE_4__.expect;
+
+  [[url, url], [url, dataUrl], [dataUrl, url], [dataUrl, dataUrl]].forEach(function (sources) {
+    test("loads the same image again (" + sources.map(function (source) {
+      return source === url ? "URL" : "data";
+    }).join(" to ") + ")", function () {
+      var image = new _native.Image();
+      return sources.reduce(function (pending, source) {
+        return pending.then(function () {
+          return new Promise(function (resolve, reject) {
+            image.onload = resolve;
+            image.onerror = reject;
+            image.src = source;
+          }).then(function () {
+            expect(image.width).to.equal(source === url ? 2 : 3);
+            expect(image.naturalWidth).to.equal(image.width);
+            expect(image.height).to.equal(1);
+            expect(image.naturalHeight).to.equal(1);
+          });
+        });
+      }, Promise.resolve());
+    });
+  });
+
+  test("reflects the assigned src immediately", function () {
+    var image = new _native.Image();
+    return new Promise(function (resolve, reject) {
+      image.onload = resolve;
+      image.onerror = reject;
+      image.src = dataUrl;
+      expect(image.src).to.equal(dataUrl);
+    }).then(function () {
+      expect(image.src).to.equal(dataUrl);
+    });
+  });
+
+  [true, false].forEach(function (fromUrl) {
+    test("only delivers the latest assignment after a pending " + (fromUrl ? "URL" : "data") + " load", function () {
+      setImageReloadTestResponse(buffer__WEBPACK_IMPORTED_MODULE_5__.Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR4nGP4z8AAQv8BD/kD/YURmXYAAAAASUVORK5CYII=", "base64"));
+      var image = new _native.Image();
+      var barrier = new _native.Image();
+      var loaded = [];
+      var errors = [];
+      return new Promise(function (resolve, reject) {
+        image.onload = function () { loaded.push(image.width); };
+        image.onerror = function (error) { errors.push(error); reject(error); };
+        barrier.onload = resolve;
+        barrier.onerror = reject;
+        image.src = fromUrl ? "image-reload-test:///image.png" : dataUrl;
+        image.src = dataUrl;
+        barrier.src = dataUrl;
+      }).then(function () {
+        expect(image.src).to.equal(dataUrl);
+        expect(loaded).to.deep.equal([3]);
+        expect(errors).to.deep.equal([]);
+        expect(image.width).to.equal(3);
+        expect(barrier.width).to.equal(3);
+      });
+    });
+  });
+
+  [false, true].forEach(function (toData) {
+    pixelTest("draws reloaded pixels instead of the cached texture (URL to " + (toData ? "data" : "URL") + ")", function () {
+      var canvas = new _native.Canvas();
+      canvas.width = 4;
+      canvas.height = 4;
+      var context = canvas.getContext("2d");
+      var image = new _native.Image();
+      function load(assign) {
+        return new Promise(function (resolve, reject) {
+          image.onload = resolve;
+          image.onerror = reject;
+          assign();
+        });
+      }
+      function expectChannel(channel) {
+        var data = context.getImageData(1, 1, 1, 1).data;
+        expect(data[channel]).to.be.greaterThan(240);
+        expect(data[1 - channel]).to.be.lessThan(20);
+        expect(data[2]).to.be.lessThan(20);
+        expect(data[3]).to.be.greaterThan(240);
+      }
+      return load(function () {
+        setImageReloadTestResponse(buffer__WEBPACK_IMPORTED_MODULE_5__.Buffer.from(redPng, "base64"));
+        image.src = "image-reload-test:///red.png";
+      }).then(function () {
+        context.drawImage(image, 0, 0, 4, 4);
+        expectChannel(0);
+        return load(function () {
+          if (toData) {
+            image.src = "data:image/png;base64," + greenPng;
+          } else {
+            setImageReloadTestResponse(buffer__WEBPACK_IMPORTED_MODULE_5__.Buffer.from(greenPng, "base64"));
+            image.src = "image-reload-test:///green.png";
+          }
+        });
+      }).then(function () {
+        context.clearRect(0, 0, 4, 4);
+        context.drawImage(image, 0, 0, 4, 4);
+        expectChannel(1);
+        context.dispose();
+        canvas.dispose();
+      }, function (error) {
+        context.dispose();
+        canvas.dispose();
+        throw error;
+      });
+    });
+  });
+
+  test("reports load errors and can recover with data and URL loads", function () {
+    var image = new _native.Image();
+    var errorCount = 0;
+    return new Promise(function (resolve, reject) {
+      image.onload = function () { reject(new Error("A missing file unexpectedly loaded")); };
+      image.onerror = function () { ++errorCount; resolve(); };
+      image.src = "app:///Assets/nonexistent-image-reload.png";
+    }).then(function () {
+      expect(errorCount).to.equal(1);
+      return [dataUrl, url].reduce(function (pending, source) {
+        return pending.then(function () {
+          return new Promise(function (resolve, reject) {
+            image.onload = resolve;
+            image.onerror = reject;
+            image.src = source;
+          }).then(function () {
+            expect(image.src).to.equal(source);
+            expect(image.width).to.equal(source === url ? 2 : 3);
+          });
+        });
+      }, Promise.resolve());
+    });
+  });
+});
 mocha.run(function (failures) {
   // Test program will wait for code to be set before exiting
   if (failures > 0) {

Apps/UnitTests/JavaScript/src/tests.javaScript.all.ts

@@ -36,6 +36,7 @@ declare const hostPlatform: string;
 declare const hasGpuRendering: boolean;
 declare const hasNativeImageLoading: boolean;
 declare const setExitCode: (code: number) => void;
+declare const setImageReloadTestResponse: (bytes: Uint8Array) => void;
 declare const skipCanvasGpuTests: boolean;
 declare const _native: any;
 
@@ -2125,6 +2126,138 @@ function hexToBytes(hex: string): Uint8Array {
   });
 });
 
+describe("Canvas image reloads", function () {
+  this.timeout(5000);
+  const test = hasNativeImageLoading ? it : it.skip;
+  const url = "app:///Assets/image-reload.png";
+  const dataUrl = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAMAAAABCAAAAAA+i0toAAAADElEQVR42mNgqP8PAAIBAX+LG2RhAAAAAElFTkSuQmCC";
+
+  for (const sources of [[url, url], [url, dataUrl], [dataUrl, url], [dataUrl, dataUrl]]) {
+    test(`loads the same image again (${sources.map(source => source === url ? "URL" : "data").join(" to ")})`, async function () {
+      const image = new _native.Image();
+      for (const source of sources) {
+        await new Promise<void>((resolve, reject) => {
+          image.onload = resolve;
+          image.onerror = reject;
+          image.src = source;
+        });
+        expect(image.width).to.equal(source === url ? 2 : 3);
+        expect(image.naturalWidth).to.equal(image.width);
+        expect(image.height).to.equal(1);
+        expect(image.naturalHeight).to.equal(1);
+      }
+    });
+  }
+
+  test("reflects the assigned src immediately", async function () {
+    const image = new _native.Image();
+    await new Promise<void>((resolve, reject) => {
+      image.onload = resolve;
+      image.onerror = reject;
+      image.src = dataUrl;
+      expect(image.src).to.equal(dataUrl);
+    });
+    expect(image.src).to.equal(dataUrl);
+  });
+
+  for (const fromUrl of [true, false]) {
+    test(`only delivers the latest assignment after a pending ${fromUrl ? "URL" : "data"} load`, async function () {
+      setImageReloadTestResponse(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR4nGP4z8AAQv8BD/kD/YURmXYAAAAASUVORK5CYII=", "base64"));
+      const image = new _native.Image();
+      const barrier = new _native.Image();
+      const loaded: number[] = [];
+      const errors: unknown[] = [];
+      await new Promise<void>((resolve, reject) => {
+        image.onload = () => { loaded.push(image.width); };
+        image.onerror = (error: unknown) => { errors.push(error); reject(error); };
+        barrier.onload = resolve;
+        barrier.onerror = reject;
+        // The in-memory URL resolver queues completion synchronously. The final data load
+        // is a runtime-queue barrier after both candidate callbacks, not a timing estimate.
+        image.src = fromUrl ? "image-reload-test:///image.png" : dataUrl;
+        image.src = dataUrl;
+        barrier.src = dataUrl;
+      });
+      expect(image.src).to.equal(dataUrl);
+      expect(loaded).to.deep.equal([3]);
+      expect(errors).to.deep.equal([]);
+      expect(image.width).to.equal(3);
+      expect(barrier.width).to.equal(3);
+    });
+  }
+
+  const pixelTest = hasNativeImageLoading && hasGpuRendering && !skipCanvasGpuTests ? it : it.skip;
+  const redPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADklEQVR42mP4z8DwH4QBEfcD/f6tu5kAAAAASUVORK5CYII=";
+  const greenPng = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADElEQVR42mNg+A+BAA/5A/3mxKLtAAAAAElFTkSuQmCC";
+  for (const toData of [false, true]) {
+    pixelTest(`draws reloaded pixels instead of the cached texture (URL to ${toData ? "data" : "URL"})`, async function () {
+      const canvas = new _native.Canvas();
+      canvas.width = 4;
+      canvas.height = 4;
+      const context = canvas.getContext("2d");
+      try {
+        const image = new _native.Image();
+        const load = (assign: () => void) => new Promise<void>((resolve, reject) => {
+          image.onload = resolve;
+          image.onerror = reject;
+          assign();
+        });
+        const sample = () => context.getImageData(1, 1, 1, 1).data;
+        const expectChannel = (data: any, channel: number) => {
+          expect(data[channel]).to.be.greaterThan(240);
+          expect(data[1 - channel]).to.be.lessThan(20);
+          expect(data[2]).to.be.lessThan(20);
+          expect(data[3]).to.be.greaterThan(240);
+        };
+
+        setImageReloadTestResponse(Buffer.from(redPng, "base64"));
+        await load(() => {
+          image.src = "image-reload-test:///red.png";
+        });
+        context.drawImage(image, 0, 0, 4, 4);
+        expectChannel(sample(), 0);
+
+        if (toData) {
+          await load(() => {
+            image.src = "data:image/png;base64," + greenPng;
+          });
+        } else {
+          setImageReloadTestResponse(Buffer.from(greenPng, "base64"));
+          await load(() => {
+            image.src = "image-reload-test:///green.png";
+          });
+        }
+        context.clearRect(0, 0, 4, 4);
+        context.drawImage(image, 0, 0, 4, 4);
+        expectChannel(sample(), 1);
+      } finally {
+        context.dispose();
+        canvas.dispose();
+      }
+    });
+  }
+
+  test("reports load errors and can recover with data and URL loads", async function () {
+    const image = new _native.Image();
+    let errorCount = 0;
+    await new Promise<void>((resolve, reject) => {
+      image.onload = () => reject(new Error("A missing file unexpectedly loaded"));
+      image.onerror = () => { ++errorCount; resolve(); };
+      image.src = "app:///Assets/nonexistent-image-reload.png";
+    });
+    expect(errorCount).to.equal(1);
+    for (const source of [dataUrl, url]) {
+      await new Promise<void>((resolve, reject) => {
+        image.onload = resolve;
+        image.onerror = reject;
+        image.src = source;
+      });
+      expect(image.src).to.equal(source);
+      expect(image.width).to.equal(source === url ? 2 : 3);
+    }
+  });
+});
+
 mocha.run((failures) => {
   // Test program will wait for code to be set before exiting
   if (failures > 0) {

Apps/UnitTests/Source/Tests.JavaScript.cpp

@@ -16,9 +16,14 @@
 #include <Babylon/Plugins/NativeMeshopt.h>
 #endif
 #include <Babylon/ScriptLoader.h>
+#ifdef HAS_NATIVE_IMAGE_LOADING
+#include <UrlLib/UrlLib.h>
+#include <gsl/util>
+#include <memory>
+#include <vector>
+#endif
 
 #include <cstdlib>
-#include <memory>
 
 extern Babylon::Graphics::Configuration g_deviceConfig;
 
@@ -107,6 +112,12 @@ TEST(JavaScript, All)
     // Change this to true to wait for the JavaScript debugger to attach (only applies to V8)
     constexpr const bool waitForDebugger = false;
 
+#ifdef HAS_NATIVE_IMAGE_LOADING
+    const auto unregisterImageTestScheme = gsl::finally([] {
+        UrlLib::UrlRequest::UnregisterSchemeResolver("image-reload-test");
+    });
+#endif
+
     Babylon::Graphics::Device device{g_deviceConfig};
 
     // Start rendering a frame to unblock the JavaScript from queuing graphics
@@ -176,6 +187,21 @@ TEST(JavaScript, All)
             },
             "setExitCode");
         env.Global().Set("setExitCode", setExitCodeCallback);
+#ifdef HAS_NATIVE_IMAGE_LOADING
+        env.Global().Set("setImageReloadTestResponse", Napi::Function::New(env, [](const Napi::CallbackInfo& info) {
+            const auto bytes = info[0].As<Napi::Uint8Array>();
+            const auto begin = reinterpret_cast<const std::byte*>(bytes.Data());
+            auto body = std::make_shared<const std::vector<std::byte>>(begin, begin + bytes.ElementLength());
+            UrlLib::UrlRequest::RegisterSchemeResolver("image-reload-test", [body](const std::string&) {
+                UrlLib::UrlSchemeResolverResult result;
+                result.handled = true;
+                result.statusCode = UrlLib::UrlStatusCode::Ok;
+                result.contentType = "image/png";
+                result.body = body;
+                return result;
+            });
+        }));
+#endif
     });
 
     Babylon::ScriptLoader loader{runtime};

Polyfills/Canvas/Source/Context.cpp

@@ -181,7 +181,7 @@ namespace Babylon::Polyfills::Internal
             ReleaseImagesAfterFlush();
             for (auto& image : m_nvgImageIndices)
             {
-                nvgDeleteImage(*m_nvg, image.second);
+                nvgDeleteImage(*m_nvg, image.second.index);
             }
             nvgDelete(*m_nvg);
             m_nvg = nullptr;
@@ -1394,15 +1394,24 @@ namespace Babylon::Polyfills::Internal
         }
 
         int imageIndex{-1};
+        const uint32_t generation = canvasImage->GetContentGeneration();
         const auto nvgImageIter = m_nvgImageIndices.find(canvasImage);
-        if (nvgImageIter == m_nvgImageIndices.end())
+        // SetBuffer keeps this NativeCanvasImage* and only replaces its pixels. A same-size
+        // reload would otherwise keep drawing the texture created from the old buffer.
+        // Hold the old handle until this frame flushes so an earlier draw can still sample it.
+        if (nvgImageIter == m_nvgImageIndices.end() || nvgImageIter->second.generation != generation)
         {
+            if (nvgImageIter != m_nvgImageIndices.end())
+            {
+                RetainImageUntilFlush(nvgImageIter->second.index);
+                m_nvgImageIndices.erase(nvgImageIter);
+            }
             imageIndex = canvasImage->CreateNVGImageForContext(*m_nvg);
-            m_nvgImageIndices.try_emplace(canvasImage, imageIndex);
+            m_nvgImageIndices.try_emplace(canvasImage, CachedNVGImage{imageIndex, generation});
         }
         else
         {
-            imageIndex = nvgImageIter->second;
+            imageIndex = nvgImageIter->second.index;
         }
         assert(imageIndex != -1);