From dd353c657166bc37f35b6c06e1b7144e672e6fc4 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 12:11:09 +0530
Subject: [PATCH 01/20] speedup binary build
- 70s to 20s
---
binary/build.js | 174 ++++++++++------------------------
binary/utils/bundle-binary.js | 77 +++++++++++++++
binary/utils/copy-lancedb.js | 82 ++++++++++++++++
3 files changed, 209 insertions(+), 124 deletions(-)
create mode 100644 binary/utils/bundle-binary.js
create mode 100644 binary/utils/copy-lancedb.js
diff --git a/binary/build.js b/binary/build.js
index b67afb94f..713ed6dc0 100644
--- a/binary/build.js
+++ b/binary/build.js
@@ -3,13 +3,10 @@ const fs = require("fs");
const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
-const {
- validateFilesPresent,
- execCmdSync,
- autodetectPlatformAndArch,
-} = require("../scripts/util");
+const { validateFilesPresent } = require("../scripts/util");
const { downloadRipgrep } = require("./utils/ripgrep");
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
+const { fork } = require("child_process");
const bin = path.join(__dirname, "bin");
const out = path.join(__dirname, "out");
@@ -29,8 +26,6 @@ function cleanSlate() {
const esbuildOutputFile = "out/index.js";
let targets = [...ALL_TARGETS];
-const [currentPlatform, currentArch] = autodetectPlatformAndArch();
-
const assetBackups = [
"node_modules/win-ca/lib/crypt32-ia32.node.bak",
"node_modules/win-ca/lib/crypt32-x64.node.bak",
@@ -78,67 +73,6 @@ async function buildWithEsbuild() {
});
}
-async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
- console.log(`Copying ${packageName} to ${toCopy}`);
- // This is a way to install only one package without npm trying to install all the dependencies
- // Create a temporary directory for installing the package
- const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
- const tempDir = path.join(
- __dirname,
- "tmp",
- `continue-node_modules-${adjustedName}`,
- );
- const currentDir = process.cwd();
-
- // // Remove the dir we will be copying to
- // rimrafSync(`node_modules/${toCopy}`);
-
- // // Ensure the temporary directory exists
- fs.mkdirSync(tempDir, { recursive: true });
-
- try {
- // Move to the temporary directory
- process.chdir(tempDir);
-
- // Initialize a new package.json and install the package
- execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
-
- console.log(
- `Contents of: ${packageName}`,
- fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
- );
-
- // Without this it seems the file isn't completely written to disk
- await new Promise((resolve) => setTimeout(resolve, 2000));
-
- // Copy the installed package back to the current directory
- await new Promise((resolve, reject) => {
- ncp(
- path.join(tempDir, "node_modules", toCopy),
- path.join(currentDir, "node_modules", toCopy),
- { dereference: true },
- (error) => {
- if (error) {
- console.error(
- `[error] Error copying ${packageName} package`,
- error,
- );
- reject(error);
- } else {
- resolve();
- }
- },
- );
- });
- } finally {
- // Clean up the temporary directory
- // rimrafSync(tempDir);
-
- // Return to the original directory
- process.chdir(currentDir);
- }
-}
-
/**
* Downloads and installs ripgrep binaries for the specified target
*
@@ -181,16 +115,35 @@ async function downloadRipgrepForTarget(target, targetDir) {
),
);
- console.log("[info] Downloading prebuilt lancedb...");
+ const copyLanceDBPromises = [];
for (const target of targets) {
- if (TARGET_TO_LANCEDB[target]) {
- console.log(`[info] Downloading for ${target}...`);
- await installNodeModuleInTempDirAndCopyToCurrent(
- TARGET_TO_LANCEDB[target],
- "@lancedb",
- );
+ if (!TARGET_TO_LANCEDB[target]) {
+ continue;
}
+ console.log(`[info] Downloading for ${target}...`);
+ const child = fork("./utils/copy-lancedb.js", { stdio: "inherit" });
+ child.send({
+ payload: {
+ packageName: TARGET_TO_LANCEDB[target],
+ toCopy: "@lancedb",
+ },
+ });
+ copyLanceDBPromises.push(
+ new Promise((resolve, reject) => {
+ child.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ }),
+ );
}
+ await Promise.all(copyLanceDBPromises).catch(() => {
+ console.error("[error] Failed to copy LanceDB");
+ process.exit(1);
+ });
+ console.log("[info] Copied all LanceDB");
// tree-sitter-wasm
const treeSitterWasmsDir = path.join(out, "tree-sitter-wasms");
@@ -252,59 +205,31 @@ async function downloadRipgrepForTarget(target, targetDir) {
"out/llamaTokenizerWorkerPool.mjs",
);
+ const buildBinaryPromises = [];
console.log("[info] Building binaries with pkg...");
for (const target of targets) {
- const targetDir = `bin/${target}`;
- fs.mkdirSync(targetDir, { recursive: true });
- console.log(`[info] Building ${target}...`);
- execCmdSync(
- `npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
+ const child = fork("./utils/bundle-binary.js", { stdio: "inherit" });
+ child.send({
+ payload: {
+ target,
+ },
+ });
+ buildBinaryPromises.push(
+ new Promise((resolve, reject) => {
+ child.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ }),
);
-
- // Download and unzip prebuilt sqlite3 binary for the target
- console.log("[info] Downloading node-sqlite3");
-
- const downloadUrl =
- // node-sqlite3 doesn't have a pre-built binary for win32-arm64
- target === "win32-arm64"
- ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
- : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
- target
- }.tar.gz`;
-
- execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
- execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
-
- // Copy to build directory for testing
- try {
- const [platform, arch] = target.split("-");
- if (platform === currentPlatform && arch === currentArch) {
- fs.copyFileSync(
- `${targetDir}/build/Release/node_sqlite3.node`,
- `build/node_sqlite3.node`,
- );
- }
- } catch (error) {
- console.log("[warn] Could not copy node_sqlite to build");
- console.log(error);
- }
-
- fs.unlinkSync(`${targetDir}/build.tar.gz`);
-
- // copy @lancedb to bin folders
- console.log("[info] Copying @lancedb files to bin");
- fs.copyFileSync(
- `node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
- `${targetDir}/index.node`,
- );
-
- // Download and install ripgrep for the target
- await downloadRipgrepForTarget(target, targetDir);
-
- // Informs the `continue-binary` of where to look for node_sqlite3.node
- // https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
- fs.writeFileSync(`${targetDir}/package.json`, "");
}
+ await Promise.all(buildBinaryPromises).catch(() => {
+ console.error("[error] Failed to build binaries");
+ process.exit(1);
+ });
+ console.log("[info] All binaries built");
// Cleanup - this is needed when running locally
fs.rmSync("out/package.json");
@@ -331,4 +256,5 @@ async function downloadRipgrepForTarget(target, targetDir) {
validateFilesPresent(pathsToVerify);
console.log("[info] Done!");
+ process.exit(0);
})();
diff --git a/binary/utils/bundle-binary.js b/binary/utils/bundle-binary.js
new file mode 100644
index 000000000..a9ff0ead7
--- /dev/null
+++ b/binary/utils/bundle-binary.js
@@ -0,0 +1,77 @@
+/**
+ * @file Builds the binary for the specified target. It is intended to run as a child process.
+ */
+const {
+ execCmdSync,
+ autodetectPlatformAndArch,
+} = require("../../scripts/util");
+const { downloadRipgrep } = require("./ripgrep");
+const { TARGET_TO_LANCEDB } = require("../utils/targets");
+const fs = require("fs");
+
+/**
+ * @param {string} target the platform to build for
+ */
+async function bundleForBinary(target) {
+ const [currentPlatform, currentArch] = autodetectPlatformAndArch();
+
+ const targetDir = `bin/${target}`;
+ fs.mkdirSync(targetDir, { recursive: true });
+ console.log(`[info] Building ${target}...`);
+ execCmdSync(
+ `npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
+ );
+
+ // Download and unzip prebuilt sqlite3 binary for the target
+ console.log("[info] Downloading node-sqlite3");
+
+ const downloadUrl =
+ // node-sqlite3 doesn't have a pre-built binary for win32-arm64
+ target === "win32-arm64"
+ ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
+ : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
+ target
+ }.tar.gz`;
+
+ execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
+ execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
+
+ // Copy to build directory for testing
+ try {
+ const [platform, arch] = target.split("-");
+ if (platform === currentPlatform && arch === currentArch) {
+ fs.copyFileSync(
+ `${targetDir}/build/Release/node_sqlite3.node`,
+ `build/node_sqlite3.node`,
+ );
+ }
+ } catch (error) {
+ console.log("[warn] Could not copy node_sqlite to build");
+ console.log(error);
+ }
+
+ fs.unlinkSync(`${targetDir}/build.tar.gz`);
+
+ // copy @lancedb to bin folders
+ console.log("[info] Copying @lancedb files to bin");
+ fs.copyFileSync(
+ `node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
+ `${targetDir}/index.node`,
+ );
+
+ // Download and install ripgrep for the target
+ await downloadRipgrep(target, targetDir);
+
+ // Informs the `continue-binary` of where to look for node_sqlite3.node
+ // https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
+ fs.writeFileSync(`${targetDir}/package.json`, "");
+}
+
+process.on("message", (msg) => {
+ bundleForBinary(msg.payload.target)
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+});
diff --git a/binary/utils/copy-lancedb.js b/binary/utils/copy-lancedb.js
new file mode 100644
index 000000000..75c728020
--- /dev/null
+++ b/binary/utils/copy-lancedb.js
@@ -0,0 +1,82 @@
+/**
+ * @file Copy lancedb to the current directory. It is intended to run as a child process.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const ncp = require("ncp").ncp;
+const { execCmdSync } = require("../../scripts/util");
+
+async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
+ console.log(`Copying ${packageName} to ${toCopy}`);
+ // This is a way to install only one package without npm trying to install all the dependencies
+ // Create a temporary directory for installing the package
+ const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
+ const tempDir = path.join(
+ __dirname,
+ "..",
+ "tmp",
+ `continue-node_modules-${adjustedName}`,
+ );
+ const currentDir = process.cwd();
+
+ // // Remove the dir we will be copying to
+ // rimrafSync(`node_modules/${toCopy}`);
+
+ // // Ensure the temporary directory exists
+ fs.mkdirSync(tempDir, { recursive: true });
+
+ try {
+ // Move to the temporary directory
+ process.chdir(tempDir);
+
+ // Initialize a new package.json and install the package
+ execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
+
+ console.log(
+ `Contents of: ${packageName}`,
+ fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
+ );
+
+ // Without this it seems the file isn't completely written to disk
+ await new Promise((resolve) => setTimeout(resolve, 2000));
+
+ // Copy the installed package back to the current directory
+ await new Promise((resolve, reject) => {
+ ncp(
+ path.join(tempDir, "node_modules", toCopy),
+ path.join(currentDir, "node_modules", toCopy),
+ { dereference: true },
+ (error) => {
+ if (error) {
+ console.error(
+ `[error] Error copying ${packageName} package`,
+ error,
+ );
+ reject(error);
+ } else {
+ resolve();
+ }
+ },
+ );
+ });
+ } finally {
+ // Clean up the temporary directory
+ // rimrafSync(tempDir);
+
+ // Return to the original directory
+ process.chdir(currentDir);
+ }
+}
+
+process.on("message", (msg) => {
+ installNodeModuleInTempDirAndCopyToCurrent(
+ msg.payload.packageName,
+ msg.payload.toCopy,
+ )
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+});
From be9b32984c42d85b403aa7e7f5ce58bd19b37859 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 12:29:40 +0530
Subject: [PATCH 02/20] parallel downloads for node-sqlite and ripgrep
- reduction by 4 seconds
---
binary/utils/bundle-binary.js | 74 +++++++++++++++++++----------------
1 file changed, 40 insertions(+), 34 deletions(-)
diff --git a/binary/utils/bundle-binary.js b/binary/utils/bundle-binary.js
index a9ff0ead7..c218166cd 100644
--- a/binary/utils/bundle-binary.js
+++ b/binary/utils/bundle-binary.js
@@ -9,12 +9,46 @@ const { downloadRipgrep } = require("./ripgrep");
const { TARGET_TO_LANCEDB } = require("../utils/targets");
const fs = require("fs");
+async function downloadNodeSqlite(target, targetDir) {
+ return new Promise((resolve) => {
+ const [currentPlatform, currentArch] = autodetectPlatformAndArch();
+
+ // Download and unzip prebuilt sqlite3 binary for the target
+ console.log("[info] Downloading node-sqlite3");
+
+ const downloadUrl =
+ // node-sqlite3 doesn't have a pre-built binary for win32-arm64
+ target === "win32-arm64"
+ ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
+ : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
+ target
+ }.tar.gz`;
+
+ execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
+ execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
+
+ // Copy to build directory for testing
+ try {
+ const [platform, arch] = target.split("-");
+ if (platform === currentPlatform && arch === currentArch) {
+ fs.copyFileSync(
+ `${targetDir}/build/Release/node_sqlite3.node`,
+ `build/node_sqlite3.node`,
+ );
+ }
+ } catch (error) {
+ console.log("[warn] Could not copy node_sqlite to build");
+ console.log(error);
+ }
+ fs.unlinkSync(`${targetDir}/build.tar.gz`);
+ resolve();
+ });
+}
+
/**
* @param {string} target the platform to build for
*/
async function bundleForBinary(target) {
- const [currentPlatform, currentArch] = autodetectPlatformAndArch();
-
const targetDir = `bin/${target}`;
fs.mkdirSync(targetDir, { recursive: true });
console.log(`[info] Building ${target}...`);
@@ -22,36 +56,6 @@ async function bundleForBinary(target) {
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
);
- // Download and unzip prebuilt sqlite3 binary for the target
- console.log("[info] Downloading node-sqlite3");
-
- const downloadUrl =
- // node-sqlite3 doesn't have a pre-built binary for win32-arm64
- target === "win32-arm64"
- ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
- : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
- target
- }.tar.gz`;
-
- execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
- execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
-
- // Copy to build directory for testing
- try {
- const [platform, arch] = target.split("-");
- if (platform === currentPlatform && arch === currentArch) {
- fs.copyFileSync(
- `${targetDir}/build/Release/node_sqlite3.node`,
- `build/node_sqlite3.node`,
- );
- }
- } catch (error) {
- console.log("[warn] Could not copy node_sqlite to build");
- console.log(error);
- }
-
- fs.unlinkSync(`${targetDir}/build.tar.gz`);
-
// copy @lancedb to bin folders
console.log("[info] Copying @lancedb files to bin");
fs.copyFileSync(
@@ -59,8 +63,10 @@ async function bundleForBinary(target) {
`${targetDir}/index.node`,
);
- // Download and install ripgrep for the target
- await downloadRipgrep(target, targetDir);
+ const downloadPromises = [];
+ downloadPromises.push(downloadRipgrep(target, targetDir));
+ downloadPromises.push(downloadNodeSqlite(target, targetDir));
+ await Promise.all(downloadPromises);
// Informs the `continue-binary` of where to look for node_sqlite3.node
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
From 8194ec700c0850cd8cbf61ab520da5d1cec96a1d Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 18:46:21 +0530
Subject: [PATCH 03/20] parallel npm installation of vscode and gui
---
.../vscode/scripts/install-nodemodules.js | 41 +++++++++++++++
extensions/vscode/scripts/prepackage.js | 19 ++++---
extensions/vscode/scripts/utils.js | 50 ++++++++++++++-----
3 files changed, 87 insertions(+), 23 deletions(-)
create mode 100644 extensions/vscode/scripts/install-nodemodules.js
diff --git a/extensions/vscode/scripts/install-nodemodules.js b/extensions/vscode/scripts/install-nodemodules.js
new file mode 100644
index 000000000..eb71f7223
--- /dev/null
+++ b/extensions/vscode/scripts/install-nodemodules.js
@@ -0,0 +1,41 @@
+/**
+ * @file Install node modules for the VS Code extension. This is intended to run as a child process.
+ */
+
+const path = require("path");
+
+const { execCmdSync } = require("../../../scripts/util");
+
+/**
+ * @param {string} continueDir
+ */
+async function installNodeModulesInGui(continueDir) {
+ process.chdir(path.join(continueDir, "gui"));
+ execCmdSync("npm install");
+ console.log("[info] npm install in gui completed");
+}
+
+async function installNodeModulesInVscode(continueDir) {
+ process.chdir(path.join(continueDir, "extensions", "vscode"));
+ execCmdSync("npm install");
+ console.log("[info] npm install in extensions/vscode completed");
+}
+
+process.on("message", (msg) => {
+ const { continueDir, targetDir } = msg.payload;
+ if (targetDir === "gui") {
+ installNodeModulesInGui(continueDir)
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+ } else if (targetDir === "vscode") {
+ installNodeModulesInVscode(continueDir)
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+ }
+});
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 01ef446c5..bda7833f6 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -14,6 +14,7 @@ const {
copyConfigSchema,
writeBuildTimestamp,
generateConfigYamlSchema,
+ installNodeModules,
} = require("./utils");
// Clear folders that will be packaged to ensure clean slate
@@ -36,10 +37,10 @@ if (args[2] === "--target") {
let os;
let arch;
-if (!target) {
- [os, arch] = autodetectPlatformAndArch();
-} else {
+if (target) {
[os, arch] = target.split("-");
+} else {
+ [os, arch] = autodetectPlatformAndArch();
}
if (os === "alpine") {
@@ -82,13 +83,9 @@ void (async () => {
writeBuildTimestamp();
// Install node_modules //
- execCmdSync("npm install");
- console.log("[info] npm install in extensions/vscode completed");
+ await installNodeModules();
- process.chdir("../../gui");
-
- execCmdSync("npm install");
- console.log("[info] npm install in gui completed");
+ process.chdir(path.join("__dirname", "..", "..", "..", "gui"));
if (isInGitHubAction) {
execCmdSync("npm run build");
@@ -465,7 +462,7 @@ void (async () => {
);
// delete esbuild/bin because platform-specific @esbuild is downloaded
- fs.rmdirSync(`out/node_modules/esbuild/bin`, { recursive: true });
+ fs.rmSync(`out/node_modules/esbuild/bin`, { recursive: true });
console.log(`[info] Copied ${NODE_MODULES_TO_COPY.join(", ")}`);
@@ -534,4 +531,6 @@ void (async () => {
`out/node_modules/@lancedb/vectordb-${target}${isWinTarget ? "-msvc" : ""}${isLinuxTarget ? "-gnu" : ""}/index.node`,
`out/node_modules/esbuild/lib/main.js`,
]);
+
+ process.exit(0);
})();
diff --git a/extensions/vscode/scripts/utils.js b/extensions/vscode/scripts/utils.js
index 451fde407..b0b4966b1 100644
--- a/extensions/vscode/scripts/utils.js
+++ b/extensions/vscode/scripts/utils.js
@@ -1,7 +1,10 @@
+const { fork } = require("child_process");
const fs = require("fs");
-const ncp = require("ncp").ncp;
const path = require("path");
+
+const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
+
const {
validateFilesPresent,
execCmdSync,
@@ -77,20 +80,41 @@ function copyTokenizers() {
console.log("[info] Copied llamaTokenizer");
}
-function installNodeModules() {
- // Make sure we are in the right directory
- if (!process.cwd().endsWith("vscode")) {
- process.chdir(path.join(continueDir, "extensions", "vscode"));
- }
+async function installNodeModules() {
+ const installVscodeChild = fork(
+ path.join(__dirname, "install-nodemodules.js"),
+ {
+ stdio: "inherit",
+ },
+ );
+ installVscodeChild.send({ payload: { continueDir, targetDir: "vscode" } });
- // Install node_modules //
- execCmdSync("npm install");
- console.log("[info] npm install in extensions/vscode completed");
+ const installGuiChild = fork(path.join(__dirname, "install-nodemodules.js"), {
+ stdio: "inherit",
+ });
+ installGuiChild.send({ payload: { continueDir, targetDir: "gui" } });
- process.chdir(path.join(continueDir, "gui"));
-
- execCmdSync("npm install");
- console.log("[info] npm install in gui completed");
+ await Promise.all([
+ new Promise((resolve, reject) => {
+ installVscodeChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ }),
+ new Promise((resolve, reject) => {
+ installGuiChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ }),
+ ]).catch((error) => {
+ console.error(error);
+ process.exit(1);
+ });
}
async function buildGui(isGhAction) {
From c0700e1634dd96cf5d1743f8d52557cd0d1ddadb Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 19:08:53 +0530
Subject: [PATCH 04/20] parallel config yaml generation copying and installing
node modules
---
.../vscode/scripts/generate-copy-config.js | 83 ++++++++++++++++++
.../vscode/scripts/install-nodemodules.js | 15 ++--
extensions/vscode/scripts/prepackage.js | 20 +----
extensions/vscode/scripts/utils.js | 87 ++++++++-----------
4 files changed, 130 insertions(+), 75 deletions(-)
create mode 100644 extensions/vscode/scripts/generate-copy-config.js
diff --git a/extensions/vscode/scripts/generate-copy-config.js b/extensions/vscode/scripts/generate-copy-config.js
new file mode 100644
index 000000000..16d8e1df6
--- /dev/null
+++ b/extensions/vscode/scripts/generate-copy-config.js
@@ -0,0 +1,83 @@
+/**
+ * @file Generate config.yaml file from template. Intended to run as a child process.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const { execCmdSync } = require("../../../scripts/util");
+
+const { continueDir } = require("./utils");
+
+async function generateConfigYamlSchema() {
+ process.chdir(path.join(continueDir, "packages", "config-yaml"));
+ execCmdSync("npm install");
+ execCmdSync("npm run build");
+ execCmdSync("npm run generate-schema");
+ fs.copyFileSync(
+ path.join("schema", "config-yaml-schema.json"),
+ path.join(continueDir, "extensions", "vscode", "config-yaml-schema.json"),
+ );
+ console.log("[info] Generated config.yaml schema");
+}
+
+async function copyConfigSchema() {
+ process.chdir(path.join(continueDir, "extensions", "vscode"));
+ // Modify and copy for .continuerc.json
+ const schema = JSON.parse(fs.readFileSync("config_schema.json", "utf8"));
+ schema.$defs.SerializedContinueConfig.properties.mergeBehavior = {
+ type: "string",
+ enum: ["merge", "overwrite"],
+ default: "merge",
+ title: "Merge behavior",
+ markdownDescription:
+ "If set to 'merge', .continuerc.json will be applied on top of config.json (arrays and objects are merged). If set to 'overwrite', then every top-level property of .continuerc.json will overwrite that property from config.json.",
+ "x-intellij-html-description":
+ "If set to merge, .continuerc.json will be applied on top of config.json (arrays and objects are merged). If set to overwrite, then every top-level property of .continuerc.json will overwrite that property from config.json.
",
+ };
+ fs.writeFileSync("continue_rc_schema.json", JSON.stringify(schema, null, 2));
+
+ // Copy config schemas to intellij
+ fs.copyFileSync(
+ "config_schema.json",
+ path.join(
+ "..",
+ "intellij",
+ "src",
+ "main",
+ "resources",
+ "config_schema.json",
+ ),
+ );
+ fs.copyFileSync(
+ "continue_rc_schema.json",
+ path.join(
+ "..",
+ "intellij",
+ "src",
+ "main",
+ "resources",
+ "continue_rc_schema.json",
+ ),
+ );
+}
+
+process.on("message", (msg) => {
+ const { operation } = msg.payload;
+ if (operation === "generate") {
+ generateConfigYamlSchema()
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+ }
+ if (operation === "copy") {
+ copyConfigSchema()
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+ }
+});
diff --git a/extensions/vscode/scripts/install-nodemodules.js b/extensions/vscode/scripts/install-nodemodules.js
index eb71f7223..cae222b39 100644
--- a/extensions/vscode/scripts/install-nodemodules.js
+++ b/extensions/vscode/scripts/install-nodemodules.js
@@ -6,32 +6,31 @@ const path = require("path");
const { execCmdSync } = require("../../../scripts/util");
-/**
- * @param {string} continueDir
- */
-async function installNodeModulesInGui(continueDir) {
+const { continueDir } = require("./utils");
+
+async function installNodeModulesInGui() {
process.chdir(path.join(continueDir, "gui"));
execCmdSync("npm install");
console.log("[info] npm install in gui completed");
}
-async function installNodeModulesInVscode(continueDir) {
+async function installNodeModulesInVscode() {
process.chdir(path.join(continueDir, "extensions", "vscode"));
execCmdSync("npm install");
console.log("[info] npm install in extensions/vscode completed");
}
process.on("message", (msg) => {
- const { continueDir, targetDir } = msg.payload;
+ const { targetDir } = msg.payload;
if (targetDir === "gui") {
- installNodeModulesInGui(continueDir)
+ installNodeModulesInGui()
.then(() => process.send({ done: true }))
.catch((error) => {
console.error(error); // show the error in the parent process
process.send({ error: true });
});
} else if (targetDir === "vscode") {
- installNodeModulesInVscode(continueDir)
+ installNodeModulesInVscode()
.then(() => process.send({ done: true }))
.catch((error) => {
console.error(error); // show the error in the parent process
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index bda7833f6..e57059261 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -11,10 +11,10 @@ const {
} = require("../../../scripts/util/index");
const {
- copyConfigSchema,
writeBuildTimestamp,
- generateConfigYamlSchema,
installNodeModules,
+ continueDir,
+ generateAndCopyConfigYamlSchema,
} = require("./utils");
// Clear folders that will be packaged to ensure clean slate
@@ -68,24 +68,12 @@ const isMacTarget = target?.startsWith("darwin");
void (async () => {
console.log("[info] Packaging extension for target ", target);
- // Generate and copy over config-yaml-schema.json
- generateConfigYamlSchema();
-
- // Copy config schemas to intellij
- copyConfigSchema();
-
- if (!process.cwd().endsWith("vscode")) {
- // This is sometimes run from root dir instead (e.g. in VS Code tasks)
- process.chdir("extensions/vscode");
- }
-
// Make sure we have an initial timestamp file
writeBuildTimestamp();
- // Install node_modules //
- await installNodeModules();
+ await Promise.all([generateAndCopyConfigYamlSchema(), installNodeModules()]);
- process.chdir(path.join("__dirname", "..", "..", "..", "gui"));
+ process.chdir(path.join(continueDir, "gui"));
if (isInGitHubAction) {
execCmdSync("npm run build");
diff --git a/extensions/vscode/scripts/utils.js b/extensions/vscode/scripts/utils.js
index b0b4966b1..422e6370b 100644
--- a/extensions/vscode/scripts/utils.js
+++ b/extensions/vscode/scripts/utils.js
@@ -13,57 +13,42 @@ const {
const continueDir = path.join(__dirname, "..", "..", "..");
-function generateConfigYamlSchema() {
- process.chdir(path.join(continueDir, "packages", "config-yaml"));
- execCmdSync("npm install");
- execCmdSync("npm run build");
- execCmdSync("npm run generate-schema");
- fs.copyFileSync(
- path.join("schema", "config-yaml-schema.json"),
- path.join(continueDir, "extensions", "vscode", "config-yaml-schema.json"),
+async function generateAndCopyConfigYamlSchema() {
+ // Generate and copy over config-yaml-schema.json
+ const generateConfigYamlChild = fork(
+ path.join(__dirname, "generate-copy-config.js"),
+ {
+ stdio: "inherit",
+ },
);
- console.log("[info] Generated config.yaml schema");
-}
+ generateConfigYamlChild.send({ payload: { operation: "generate" } });
-function copyConfigSchema() {
- process.chdir(path.join(continueDir, "extensions", "vscode"));
- // Modify and copy for .continuerc.json
- const schema = JSON.parse(fs.readFileSync("config_schema.json", "utf8"));
- schema.$defs.SerializedContinueConfig.properties.mergeBehavior = {
- type: "string",
- enum: ["merge", "overwrite"],
- default: "merge",
- title: "Merge behavior",
- markdownDescription:
- "If set to 'merge', .continuerc.json will be applied on top of config.json (arrays and objects are merged). If set to 'overwrite', then every top-level property of .continuerc.json will overwrite that property from config.json.",
- "x-intellij-html-description":
- "If set to merge, .continuerc.json will be applied on top of config.json (arrays and objects are merged). If set to overwrite, then every top-level property of .continuerc.json will overwrite that property from config.json.
",
- };
- fs.writeFileSync("continue_rc_schema.json", JSON.stringify(schema, null, 2));
+ await new Promise((resolve, reject) => {
+ generateConfigYamlChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
// Copy config schemas to intellij
- fs.copyFileSync(
- "config_schema.json",
- path.join(
- "..",
- "intellij",
- "src",
- "main",
- "resources",
- "config_schema.json",
- ),
- );
- fs.copyFileSync(
- "continue_rc_schema.json",
- path.join(
- "..",
- "intellij",
- "src",
- "main",
- "resources",
- "continue_rc_schema.json",
- ),
+ const copyConfigSchemaChild = fork(
+ path.join(__dirname, "generate-copy-config.js"),
+ {
+ stdio: "inherit",
+ },
);
+ copyConfigSchemaChild.send({ payload: { operation: "copy" } });
+
+ await new Promise((resolve, reject) => {
+ copyConfigSchemaChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
}
function copyTokenizers() {
@@ -87,12 +72,12 @@ async function installNodeModules() {
stdio: "inherit",
},
);
- installVscodeChild.send({ payload: { continueDir, targetDir: "vscode" } });
+ installVscodeChild.send({ payload: { targetDir: "vscode" } });
const installGuiChild = fork(path.join(__dirname, "install-nodemodules.js"), {
stdio: "inherit",
});
- installGuiChild.send({ payload: { continueDir, targetDir: "gui" } });
+ installGuiChild.send({ payload: { targetDir: "gui" } });
await Promise.all([
new Promise((resolve, reject) => {
@@ -560,14 +545,14 @@ async function copyScripts() {
// in the build
function writeBuildTimestamp() {
fs.writeFileSync(
- "src/.buildTimestamp.ts",
+ path.join(continueDir, "extensions/vscode", "src/.buildTimestamp.ts"),
`export default "${new Date().toISOString()}";\n`,
);
}
module.exports = {
- generateConfigYamlSchema,
- copyConfigSchema,
+ continueDir,
+ generateAndCopyConfigYamlSchema,
installNodeModules,
buildGui,
copyOnnxRuntimeFromNodeModules,
From 112bdcb0ef0d068663271111cf47cbd34f30751a Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 20:29:00 +0530
Subject: [PATCH 05/20] move copy-lancedb to root scripts/util
---
binary/build.js | 39 +++----------------
package.json | 1 +
.../utils => scripts/util}/copy-lancedb.js | 36 +++++++++++++++--
3 files changed, 38 insertions(+), 38 deletions(-)
rename {binary/utils => scripts/util}/copy-lancedb.js (76%)
diff --git a/binary/build.js b/binary/build.js
index 713ed6dc0..c88ac82cc 100644
--- a/binary/build.js
+++ b/binary/build.js
@@ -4,9 +4,12 @@ const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
const { validateFilesPresent } = require("../scripts/util");
-const { downloadRipgrep } = require("./utils/ripgrep");
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
const { fork } = require("child_process");
+const {
+ copyLanceDBFilePath,
+ copyLanceDB,
+} = require("../scripts/util/copy-lancedb");
const bin = path.join(__dirname, "bin");
const out = path.join(__dirname, "out");
@@ -73,24 +76,6 @@ async function buildWithEsbuild() {
});
}
-/**
- * Downloads and installs ripgrep binaries for the specified target
- *
- * @param {string} target - Target platform-arch (e.g., 'darwin-x64')
- * @param {string} targetDir - Directory to install ripgrep to
- * @returns {Promise}
- */
-async function downloadRipgrepForTarget(target, targetDir) {
- console.log(`[info] Downloading ripgrep for ${target}...`);
- try {
- await downloadRipgrep(target, targetDir);
- console.log(`[info] Successfully installed ripgrep for ${target}`);
- } catch (error) {
- console.error(`[error] Failed to download ripgrep for ${target}:`, error);
- throw error;
- }
-}
-
(async () => {
if (esbuildOnly) {
await buildWithEsbuild();
@@ -121,22 +106,8 @@ async function downloadRipgrepForTarget(target, targetDir) {
continue;
}
console.log(`[info] Downloading for ${target}...`);
- const child = fork("./utils/copy-lancedb.js", { stdio: "inherit" });
- child.send({
- payload: {
- packageName: TARGET_TO_LANCEDB[target],
- toCopy: "@lancedb",
- },
- });
copyLanceDBPromises.push(
- new Promise((resolve, reject) => {
- child.on("message", (msg) => {
- if (msg.error) {
- reject();
- }
- resolve();
- });
- }),
+ copyLanceDB(TARGET_TO_LANCEDB[target], "@lancedb"),
);
}
await Promise.all(copyLanceDBPromises).catch(() => {
diff --git a/package.json b/package.json
index 79e522af1..9100ad0cd 100644
--- a/package.json
+++ b/package.json
@@ -12,6 +12,7 @@
"@typescript-eslint/parser": "^7.8.0",
"concurrently": "^9.1.2",
"eslint-plugin-import": "^2.29.1",
+ "ncp": "^2.0.0",
"prettier": "^3.3.3",
"prettier-plugin-tailwindcss": "^0.6.8",
"typescript": "^5.6.3"
diff --git a/binary/utils/copy-lancedb.js b/scripts/util/copy-lancedb.js
similarity index 76%
rename from binary/utils/copy-lancedb.js
rename to scripts/util/copy-lancedb.js
index 75c728020..dc520a483 100644
--- a/binary/utils/copy-lancedb.js
+++ b/scripts/util/copy-lancedb.js
@@ -5,20 +5,20 @@
const fs = require("fs");
const path = require("path");
const ncp = require("ncp").ncp;
-const { execCmdSync } = require("../../scripts/util");
+const { execCmdSync } = require(".");
+const { fork } = require("child_process");
async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
console.log(`Copying ${packageName} to ${toCopy}`);
// This is a way to install only one package without npm trying to install all the dependencies
// Create a temporary directory for installing the package
const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
+ const currentDir = process.cwd();
const tempDir = path.join(
- __dirname,
- "..",
+ currentDir,
"tmp",
`continue-node_modules-${adjustedName}`,
);
- const currentDir = process.cwd();
// // Remove the dir we will be copying to
// rimrafSync(`node_modules/${toCopy}`);
@@ -80,3 +80,31 @@ process.on("message", (msg) => {
process.send({ error: true });
});
});
+
+/**
+ * invoke a child process to install and copy lancedb into node modules
+ * @param {string} packageName the lancedb platform to install and copy
+ * @param {string} toCopy directory to copy into inside node modules
+ */
+async function copyLanceDB(packageName, toCopy) {
+ const child = fork(__filename, { stdio: "inherit", cwd: process.cwd() });
+ child.send({
+ payload: {
+ packageName,
+ toCopy,
+ },
+ });
+
+ return new Promise((resolve, reject) => {
+ child.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
+}
+
+module.exports = {
+ copyLanceDB,
+};
From 37139c9e11b1d2e9dfa81bf531cb2bc96e7fb975 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 20:39:03 +0530
Subject: [PATCH 06/20] parallel node sqlite and esbuild downloads
---
extensions/vscode/.gitignore | 1 +
.../scripts/install-copy-sqlite-esbuild.js | 107 ++++++++++++++++++
extensions/vscode/scripts/prepackage.js | 103 ++---------------
3 files changed, 116 insertions(+), 95 deletions(-)
create mode 100644 extensions/vscode/scripts/install-copy-sqlite-esbuild.js
diff --git a/extensions/vscode/.gitignore b/extensions/vscode/.gitignore
index 3ba3940c3..8b752d1ad 100644
--- a/extensions/vscode/.gitignore
+++ b/extensions/vscode/.gitignore
@@ -9,6 +9,7 @@ src/client
exe
bin
assets
+tmp
gui/**
diff --git a/extensions/vscode/scripts/install-copy-sqlite-esbuild.js b/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
new file mode 100644
index 000000000..502001649
--- /dev/null
+++ b/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
@@ -0,0 +1,107 @@
+const { fork, execSync } = require("child_process");
+const fs = require("fs");
+
+const { rimrafSync } = require("rimraf");
+
+const { execCmdSync } = require("../../../scripts/util");
+
+async function installAndCopySqlite(target) {
+ // Replace the installed with pre-built
+ console.log("[info] Downloading pre-built sqlite3 binary");
+ rimrafSync("../../core/node_modules/sqlite3/build");
+ const downloadUrl = {
+ "darwin-arm64":
+ "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-darwin-arm64.tar.gz",
+ "linux-arm64":
+ "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v3-linux-arm64.tar.gz",
+ // node-sqlite3 doesn't have a pre-built binary for win32-arm64
+ "win32-arm64":
+ "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz",
+ }[target];
+ execCmdSync(
+ `curl -L -o ../../core/node_modules/sqlite3/build.tar.gz ${downloadUrl}`,
+ );
+ execCmdSync("cd ../../core/node_modules/sqlite3 && tar -xvzf build.tar.gz");
+ fs.unlinkSync("../../core/node_modules/sqlite3/build.tar.gz");
+}
+
+async function installAndCopyEsbuild(target) {
+ // Download and unzip esbuild
+ console.log("[info] Downloading pre-built esbuild binary");
+ rimrafSync("node_modules/@esbuild");
+ fs.mkdirSync("node_modules/@esbuild", { recursive: true });
+ execCmdSync(
+ `curl -o node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${target}/esbuild.zip`,
+ );
+ execCmdSync("cd node_modules/@esbuild && unzip esbuild.zip");
+ fs.unlinkSync("node_modules/@esbuild/esbuild.zip");
+}
+
+process.on("message", (msg) => {
+ const { operation, target } = msg.payload;
+ if (operation === "sqlite") {
+ installAndCopySqlite(target)
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+ }
+ if (operation === "esbuild") {
+ installAndCopyEsbuild(target)
+ .then(() => process.send({ done: true }))
+ .catch((error) => {
+ console.error(error); // show the error in the parent process
+ process.send({ error: true });
+ });
+ }
+});
+
+/**
+ * @param {string} target the platform to build for
+ */
+async function copySqlite(target) {
+ const child = fork(__filename, { stdio: "inherit", cwd: process.cwd() });
+ child.send({
+ payload: {
+ operation: "sqlite",
+ target,
+ },
+ });
+
+ return new Promise((resolve, reject) => {
+ child.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
+}
+
+/**
+ * @param {string} target the platform to build for
+ */
+async function copyEsbuild(target) {
+ const child = fork(__filename, { stdio: "inherit", cwd: process.cwd() });
+ child.send({
+ payload: {
+ operation: "esbuild",
+ target,
+ },
+ });
+
+ return new Promise((resolve, reject) => {
+ child.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
+}
+
+module.exports = {
+ copySqlite,
+ copyEsbuild,
+};
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index e57059261..43fe574f0 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -4,12 +4,14 @@ const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
+const { copyLanceDB } = require("../../../scripts/util/copy-lancedb");
const {
validateFilesPresent,
execCmdSync,
autodetectPlatformAndArch,
} = require("../../../scripts/util/index");
+const { copySqlite, copyEsbuild } = require("./install-copy-sqlite-esbuild");
const {
writeBuildTimestamp,
installNodeModules,
@@ -265,68 +267,6 @@ void (async () => {
);
});
- async function installNodeModuleInTempDirAndCopyToCurrent(
- packageName,
- toCopy,
- ) {
- console.log(`Copying ${packageName} to ${toCopy}`);
- // This is a way to install only one package without npm trying to install all the dependencies
- // Create a temporary directory for installing the package
- const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
-
- const tempDir = `/tmp/continue-node_modules-${adjustedName}`;
- const currentDir = process.cwd();
-
- // Remove the dir we will be copying to
- rimrafSync(`node_modules/${toCopy}`);
-
- // Ensure the temporary directory exists
- fs.mkdirSync(tempDir, { recursive: true });
-
- try {
- // Move to the temporary directory
- process.chdir(tempDir);
-
- // Initialize a new package.json and install the package
- execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
-
- console.log(
- `Contents of: ${packageName}`,
- fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
- );
-
- // Without this it seems the file isn't completely written to disk
- // Ideally we validate file integrity in the validation at the end
- await new Promise((resolve) => setTimeout(resolve, 2000));
-
- // Copy the installed package back to the current directory
- await new Promise((resolve, reject) => {
- ncp(
- path.join(tempDir, "node_modules", toCopy),
- path.join(currentDir, "node_modules", toCopy),
- { dereference: true },
- (error) => {
- if (error) {
- console.error(
- `[error] Error copying ${packageName} package`,
- error,
- );
- reject(error);
- } else {
- resolve();
- }
- },
- );
- });
- } finally {
- // Clean up the temporary directory
- // rimrafSync(tempDir);
-
- // Return to the original directory
- process.chdir(currentDir);
- }
- }
-
// GitHub Actions doesn't support ARM, so we need to download pre-saved binaries
// 02/07/25 - the above comment is out of date, there is now support for ARM runners on GitHub Actions
if (isArmTarget) {
@@ -340,41 +280,14 @@ void (async () => {
"[info] Downloading pre-built lancedb binary: " + packageToInstall,
);
- await installNodeModuleInTempDirAndCopyToCurrent(
- packageToInstall,
- "@lancedb",
- );
-
- // Replace the installed with pre-built
- console.log("[info] Downloading pre-built sqlite3 binary");
- rimrafSync("../../core/node_modules/sqlite3/build");
- const downloadUrl = {
- "darwin-arm64":
- "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-darwin-arm64.tar.gz",
- "linux-arm64":
- "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v3-linux-arm64.tar.gz",
- // node-sqlite3 doesn't have a pre-built binary for win32-arm64
- "win32-arm64":
- "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz",
- }[target];
- execCmdSync(
- `curl -L -o ../../core/node_modules/sqlite3/build.tar.gz ${downloadUrl}`,
- );
- execCmdSync("cd ../../core/node_modules/sqlite3 && tar -xvzf build.tar.gz");
- fs.unlinkSync("../../core/node_modules/sqlite3/build.tar.gz");
-
- // Download and unzip esbuild
- console.log("[info] Downloading pre-built esbuild binary");
- rimrafSync("node_modules/@esbuild");
- fs.mkdirSync("node_modules/@esbuild", { recursive: true });
- execCmdSync(
- `curl -o node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${target}/esbuild.zip`,
- );
- execCmdSync(`cd node_modules/@esbuild && unzip esbuild.zip`);
- fs.unlinkSync("node_modules/@esbuild/esbuild.zip");
+ await Promise.all([
+ copyLanceDB(packageToInstall, "@lancedb"),
+ copySqlite(target),
+ copyEsbuild(target),
+ ]);
} else {
// Download esbuild from npm in tmp and copy over
- console.log("npm installing esbuild binary");
+ console.log("[info] npm installing esbuild binary");
await installNodeModuleInTempDirAndCopyToCurrent(
"esbuild@0.17.19",
"@esbuild",
From e98307aafa83e62c07d49ce0c6abffbff3f9fc33 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 21:08:33 +0530
Subject: [PATCH 07/20] download using node https over curl
---
.../scripts/install-copy-sqlite-esbuild.js | 85 ++++++++++++++++++-
extensions/vscode/scripts/prepackage.js | 4 +-
2 files changed, 83 insertions(+), 6 deletions(-)
diff --git a/extensions/vscode/scripts/install-copy-sqlite-esbuild.js b/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
index 502001649..370876c7f 100644
--- a/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
+++ b/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
@@ -1,10 +1,85 @@
const { fork, execSync } = require("child_process");
const fs = require("fs");
+const https = require("https");
+const path = require("path");
const { rimrafSync } = require("rimraf");
const { execCmdSync } = require("../../../scripts/util");
+/**
+ * download a file using nodejs http
+ * @param {string} url
+ * @param {string} outputPath
+ * @param {number} maxRedirects
+ */
+async function downloadFile(url, outputPath, maxRedirects = 5) {
+ return new Promise((resolve, reject) => {
+ const downloadWithRedirects = (currentUrl, redirectCount = 0) => {
+ if (redirectCount > maxRedirects) {
+ return reject(new Error(`Too many redirects (${maxRedirects})`));
+ }
+
+ const request = https.get(currentUrl, (response) => {
+ if (
+ response.statusCode >= 300 &&
+ response.statusCode < 400 &&
+ response.headers.location
+ ) {
+ return downloadWithRedirects(
+ response.headers.location,
+ redirectCount + 1,
+ );
+ }
+
+ if (response.statusCode !== 200) {
+ return reject(
+ new Error(`HTTP ${response.statusCode}: ${response.statusMessage}`),
+ );
+ }
+
+ const outputDir = path.dirname(outputPath);
+ if (!fs.existsSync(outputDir)) {
+ fs.mkdirSync(outputDir, { recursive: true });
+ }
+
+ const writeStream = fs.createWriteStream(outputPath);
+
+ const totalSize = parseInt(response.headers["content-length"], 10);
+ let downloadedSize = 0;
+
+ response.on("data", (chunk) => {
+ downloadedSize += chunk.length;
+ if (totalSize) {
+ const percent = ((downloadedSize / totalSize) * 100).toFixed(1);
+ process.stdout.write(
+ `\rDownloading: ${percent}% (${downloadedSize}/${totalSize} bytes)`,
+ );
+ }
+ });
+
+ response.pipe(writeStream);
+
+ writeStream.on("finish", () => {
+ console.log(`\nDownload completed: ${outputPath}`);
+ resolve(outputPath);
+ });
+
+ writeStream.on("error", reject);
+ response.on("error", reject);
+ });
+
+ request.on("error", reject);
+ request.setTimeout(30000, () => {
+ request.destroy();
+ reject(new Error("Request timeout"));
+ });
+ };
+
+ downloadWithRedirects(url);
+ });
+}
+
async function installAndCopySqlite(target) {
// Replace the installed with pre-built
console.log("[info] Downloading pre-built sqlite3 binary");
@@ -18,8 +93,9 @@ async function installAndCopySqlite(target) {
"win32-arm64":
"https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz",
}[target];
- execCmdSync(
- `curl -L -o ../../core/node_modules/sqlite3/build.tar.gz ${downloadUrl}`,
+ await downloadFile(
+ downloadUrl,
+ "../../core/node_modules/sqlite3/build.tar.gz",
);
execCmdSync("cd ../../core/node_modules/sqlite3 && tar -xvzf build.tar.gz");
fs.unlinkSync("../../core/node_modules/sqlite3/build.tar.gz");
@@ -30,8 +106,9 @@ async function installAndCopyEsbuild(target) {
console.log("[info] Downloading pre-built esbuild binary");
rimrafSync("node_modules/@esbuild");
fs.mkdirSync("node_modules/@esbuild", { recursive: true });
- execCmdSync(
- `curl -o node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${target}/esbuild.zip`,
+ await downloadFile(
+ `https://continue-server-binaries.s3.us-west-1.amazonaws.com/${target}/esbuild.zip`,
+ "node_modules/@esbuild/esbuild.zip",
);
execCmdSync("cd node_modules/@esbuild && unzip esbuild.zip");
fs.unlinkSync("node_modules/@esbuild/esbuild.zip");
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 43fe574f0..6e72270e7 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -281,9 +281,9 @@ void (async () => {
);
await Promise.all([
- copyLanceDB(packageToInstall, "@lancedb"),
- copySqlite(target),
copyEsbuild(target),
+ copySqlite(target),
+ copyLanceDB(packageToInstall, "@lancedb"),
]);
} else {
// Download esbuild from npm in tmp and copy over
From 8d48cefc2e8fbd5cd011da4160a76b873875b88a Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 21:20:16 +0530
Subject: [PATCH 08/20] correct to copy-nodemodule.js
---
binary/build.js | 7 ++-----
extensions/vscode/scripts/prepackage.js | 9 +++------
scripts/util/{copy-lancedb.js => copy-nodemodule.js} | 8 ++++----
3 files changed, 9 insertions(+), 15 deletions(-)
rename scripts/util/{copy-lancedb.js => copy-nodemodule.js} (92%)
diff --git a/binary/build.js b/binary/build.js
index c88ac82cc..aa5fa5ffd 100644
--- a/binary/build.js
+++ b/binary/build.js
@@ -6,10 +6,7 @@ const { rimrafSync } = require("rimraf");
const { validateFilesPresent } = require("../scripts/util");
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
const { fork } = require("child_process");
-const {
- copyLanceDBFilePath,
- copyLanceDB,
-} = require("../scripts/util/copy-lancedb");
+const { copyNodeModule } = require("../scripts/util/copy-nodemodule");
const bin = path.join(__dirname, "bin");
const out = path.join(__dirname, "out");
@@ -107,7 +104,7 @@ async function buildWithEsbuild() {
}
console.log(`[info] Downloading for ${target}...`);
copyLanceDBPromises.push(
- copyLanceDB(TARGET_TO_LANCEDB[target], "@lancedb"),
+ copyNodeModule(TARGET_TO_LANCEDB[target], "@lancedb"),
);
}
await Promise.all(copyLanceDBPromises).catch(() => {
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 6e72270e7..ece5d91e5 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -4,7 +4,7 @@ const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
-const { copyLanceDB } = require("../../../scripts/util/copy-lancedb");
+const { copyNodeModule } = require("../../../scripts/util/copy-nodemodule");
const {
validateFilesPresent,
execCmdSync,
@@ -283,15 +283,12 @@ void (async () => {
await Promise.all([
copyEsbuild(target),
copySqlite(target),
- copyLanceDB(packageToInstall, "@lancedb"),
+ copyNodeModule(packageToInstall, "@lancedb"),
]);
} else {
// Download esbuild from npm in tmp and copy over
console.log("[info] npm installing esbuild binary");
- await installNodeModuleInTempDirAndCopyToCurrent(
- "esbuild@0.17.19",
- "@esbuild",
- );
+ await copyNodeModule("esbuild@0.17.19", "@esbuild");
}
console.log("[info] Copying sqlite node binding from core");
diff --git a/scripts/util/copy-lancedb.js b/scripts/util/copy-nodemodule.js
similarity index 92%
rename from scripts/util/copy-lancedb.js
rename to scripts/util/copy-nodemodule.js
index dc520a483..361e2d891 100644
--- a/scripts/util/copy-lancedb.js
+++ b/scripts/util/copy-nodemodule.js
@@ -82,11 +82,11 @@ process.on("message", (msg) => {
});
/**
- * invoke a child process to install and copy lancedb into node modules
- * @param {string} packageName the lancedb platform to install and copy
+ * invoke a child process to install a node module into temporary directory and copy it over into node modules
+ * @param {string} packageName the module to install and copy
* @param {string} toCopy directory to copy into inside node modules
*/
-async function copyLanceDB(packageName, toCopy) {
+async function copyNodeModule(packageName, toCopy) {
const child = fork(__filename, { stdio: "inherit", cwd: process.cwd() });
child.send({
payload: {
@@ -106,5 +106,5 @@ async function copyLanceDB(packageName, toCopy) {
}
module.exports = {
- copyLanceDB,
+ copyNodeModule,
};
From cc21abf6c236906e475efb27e79a462366d47022 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 21:27:21 +0530
Subject: [PATCH 09/20] update root package-lock
---
package-lock.json | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/package-lock.json b/package-lock.json
index 06f811afa..43530562e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,6 +8,7 @@
"@typescript-eslint/parser": "^7.8.0",
"concurrently": "^9.1.2",
"eslint-plugin-import": "^2.29.1",
+ "ncp": "^2.0.0",
"prettier": "^3.3.3",
"prettier-plugin-tailwindcss": "^0.6.8",
"typescript": "^5.6.3"
@@ -2233,6 +2234,16 @@
"dev": true,
"peer": true
},
+ "node_modules/ncp": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
+ "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==",
+ "dev": true,
+ "license": "MIT",
+ "bin": {
+ "ncp": "bin/ncp"
+ }
+ },
"node_modules/object-inspect": {
"version": "1.13.2",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz",
From 6ca088fdae20e63559cbd377f4ea51b4ccc7dd58 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 21:30:10 +0530
Subject: [PATCH 10/20] change hash to rebuild root node modules
---
.github/workflows/pr_checks.yaml | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/pr_checks.yaml b/.github/workflows/pr_checks.yaml
index 25a1b9a1e..a61958dea 100644
--- a/.github/workflows/pr_checks.yaml
+++ b/.github/workflows/pr_checks.yaml
@@ -24,7 +24,7 @@ jobs:
id: root-cache
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- name: Install root dependencies
if: steps.root-cache.outputs.cache-hit != 'true'
@@ -43,7 +43,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- name: Check code formatting with Prettier
run: npx prettier --check "**/*.{js,jsx,ts,tsx,json,css,md}" --ignore-path .gitignore --ignore-path .prettierignore
@@ -61,7 +61,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
id: config-yaml-cache
@@ -88,7 +88,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
id: config-yaml-cache
@@ -118,7 +118,7 @@ jobs:
id: root-cache
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
id: core-cache
@@ -146,7 +146,7 @@ jobs:
id: root-cache
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
id: core-cache
@@ -186,7 +186,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
with:
@@ -229,7 +229,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
with:
@@ -272,7 +272,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
with:
@@ -320,7 +320,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
with:
@@ -365,7 +365,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
- uses: actions/cache@v4
with:
From 18f1c67e20086bdb68cfb51ea4b4314106f491e0 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 21:38:08 +0530
Subject: [PATCH 11/20] Revert "change hash to rebuild root node modules"
This reverts commit 6ca088fdae20e63559cbd377f4ea51b4ccc7dd58.
---
.github/workflows/pr_checks.yaml | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/pr_checks.yaml b/.github/workflows/pr_checks.yaml
index a61958dea..25a1b9a1e 100644
--- a/.github/workflows/pr_checks.yaml
+++ b/.github/workflows/pr_checks.yaml
@@ -24,7 +24,7 @@ jobs:
id: root-cache
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- name: Install root dependencies
if: steps.root-cache.outputs.cache-hit != 'true'
@@ -43,7 +43,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- name: Check code formatting with Prettier
run: npx prettier --check "**/*.{js,jsx,ts,tsx,json,css,md}" --ignore-path .gitignore --ignore-path .prettierignore
@@ -61,7 +61,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
id: config-yaml-cache
@@ -88,7 +88,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
id: config-yaml-cache
@@ -118,7 +118,7 @@ jobs:
id: root-cache
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
id: core-cache
@@ -146,7 +146,7 @@ jobs:
id: root-cache
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
id: core-cache
@@ -186,7 +186,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
with:
@@ -229,7 +229,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
with:
@@ -272,7 +272,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
with:
@@ -320,7 +320,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
with:
@@ -365,7 +365,7 @@ jobs:
- uses: actions/cache@v4
with:
path: node_modules
- key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}-2
+ key: ${{ runner.os }}-root-node-modules-${{ hashFiles('package-lock.json') }}
- uses: actions/cache@v4
with:
From 9fae1a2f05f9c88e7a68ce360127db1174ef0e52 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:30:40 +0530
Subject: [PATCH 12/20] refactor utils to child process files
---
binary/package-lock.json | 5 +-
extensions/vscode/package-lock.json | 6 +-
.../vscode/scripts/generate-copy-config.js | 43 ++++++++++
.../vscode/scripts/install-nodemodules.js | 42 ++++++++++
extensions/vscode/scripts/prepackage.js | 9 +-
extensions/vscode/scripts/utils.js | 84 +------------------
gui/package-lock.json | 2 +-
7 files changed, 96 insertions(+), 95 deletions(-)
diff --git a/binary/package-lock.json b/binary/package-lock.json
index abb11ce8e..2d84672f0 100644
--- a/binary/package-lock.json
+++ b/binary/package-lock.json
@@ -52,7 +52,7 @@
"@aws-sdk/credential-providers": "^3.778.0",
"@continuedev/config-types": "^1.0.13",
"@continuedev/config-yaml": "file:../packages/config-yaml",
- "@continuedev/fetch": "^1.0.10",
+ "@continuedev/fetch": "^1.0.13",
"@continuedev/llm-info": "^1.0.8",
"@continuedev/openai-adapters": "^1.0.25",
"@modelcontextprotocol/sdk": "^1.12.0",
@@ -144,7 +144,8 @@
"onnxruntime-common": "1.14.0",
"onnxruntime-web": "1.14.0",
"ts-jest": "^29.1.1",
- "typescript": "^5.6.3"
+ "typescript": "^5.6.3",
+ "vitest": "^3.1.4"
},
"engines": {
"node": ">=20.19.0"
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index e516f171e..38efaa08a 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "continue",
- "version": "1.1.41",
+ "version": "1.1.44",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "continue",
- "version": "1.1.41",
+ "version": "1.1.44",
"license": "Apache-2.0",
"dependencies": {
"@continuedev/config-types": "^1.0.14",
@@ -107,7 +107,7 @@
"@aws-sdk/credential-providers": "^3.778.0",
"@continuedev/config-types": "^1.0.13",
"@continuedev/config-yaml": "file:../packages/config-yaml",
- "@continuedev/fetch": "^1.0.10",
+ "@continuedev/fetch": "^1.0.13",
"@continuedev/llm-info": "^1.0.8",
"@continuedev/openai-adapters": "^1.0.25",
"@modelcontextprotocol/sdk": "^1.12.0",
diff --git a/extensions/vscode/scripts/generate-copy-config.js b/extensions/vscode/scripts/generate-copy-config.js
index 16d8e1df6..73fe4602d 100644
--- a/extensions/vscode/scripts/generate-copy-config.js
+++ b/extensions/vscode/scripts/generate-copy-config.js
@@ -2,6 +2,7 @@
* @file Generate config.yaml file from template. Intended to run as a child process.
*/
+const { fork } = require("child_process");
const fs = require("fs");
const path = require("path");
@@ -81,3 +82,45 @@ process.on("message", (msg) => {
});
}
});
+
+async function generateAndCopyConfigYamlSchema() {
+ // Generate and copy over config-yaml-schema.json
+ const generateConfigYamlChild = fork(
+ path.join(__dirname, "generate-copy-config.js"),
+ {
+ stdio: "inherit",
+ },
+ );
+ generateConfigYamlChild.send({ payload: { operation: "generate" } });
+
+ await new Promise((resolve, reject) => {
+ generateConfigYamlChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
+
+ // Copy config schemas to intellij
+ const copyConfigSchemaChild = fork(
+ path.join(__dirname, "generate-copy-config.js"),
+ {
+ stdio: "inherit",
+ },
+ );
+ copyConfigSchemaChild.send({ payload: { operation: "copy" } });
+
+ await new Promise((resolve, reject) => {
+ copyConfigSchemaChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
+}
+
+module.exports = {
+ generateAndCopyConfigYamlSchema,
+};
diff --git a/extensions/vscode/scripts/install-nodemodules.js b/extensions/vscode/scripts/install-nodemodules.js
index cae222b39..f23de3b3b 100644
--- a/extensions/vscode/scripts/install-nodemodules.js
+++ b/extensions/vscode/scripts/install-nodemodules.js
@@ -2,6 +2,7 @@
* @file Install node modules for the VS Code extension. This is intended to run as a child process.
*/
+const { fork } = require("child_process");
const path = require("path");
const { execCmdSync } = require("../../../scripts/util");
@@ -38,3 +39,44 @@ process.on("message", (msg) => {
});
}
});
+
+async function installNodeModules() {
+ const installVscodeChild = fork(
+ path.join(__dirname, "install-nodemodules.js"),
+ {
+ stdio: "inherit",
+ },
+ );
+ installVscodeChild.send({ payload: { targetDir: "vscode" } });
+
+ const installGuiChild = fork(path.join(__dirname, "install-nodemodules.js"), {
+ stdio: "inherit",
+ });
+ installGuiChild.send({ payload: { targetDir: "gui" } });
+
+ await Promise.all([
+ new Promise((resolve, reject) => {
+ installVscodeChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ }),
+ new Promise((resolve, reject) => {
+ installGuiChild.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ }),
+ ]).catch((error) => {
+ console.error(error);
+ process.exit(1);
+ });
+}
+
+module.exports = {
+ installNodeModules,
+};
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index ece5d91e5..622393a19 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -11,13 +11,10 @@ const {
autodetectPlatformAndArch,
} = require("../../../scripts/util/index");
+const { generateAndCopyConfigYamlSchema } = require("./generate-copy-config");
const { copySqlite, copyEsbuild } = require("./install-copy-sqlite-esbuild");
-const {
- writeBuildTimestamp,
- installNodeModules,
- continueDir,
- generateAndCopyConfigYamlSchema,
-} = require("./utils");
+const { installNodeModules } = require("./install-nodemodules");
+const { writeBuildTimestamp, continueDir } = require("./utils");
// Clear folders that will be packaged to ensure clean slate
rimrafSync(path.join(__dirname, "..", "bin"));
diff --git a/extensions/vscode/scripts/utils.js b/extensions/vscode/scripts/utils.js
index 422e6370b..cae982a97 100644
--- a/extensions/vscode/scripts/utils.js
+++ b/extensions/vscode/scripts/utils.js
@@ -1,56 +1,13 @@
-const { fork } = require("child_process");
const fs = require("fs");
const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
-const {
- validateFilesPresent,
- execCmdSync,
- autodetectPlatformAndArch,
-} = require("../../../scripts/util/index");
+const { execCmdSync } = require("../../../scripts/util/index");
const continueDir = path.join(__dirname, "..", "..", "..");
-async function generateAndCopyConfigYamlSchema() {
- // Generate and copy over config-yaml-schema.json
- const generateConfigYamlChild = fork(
- path.join(__dirname, "generate-copy-config.js"),
- {
- stdio: "inherit",
- },
- );
- generateConfigYamlChild.send({ payload: { operation: "generate" } });
-
- await new Promise((resolve, reject) => {
- generateConfigYamlChild.on("message", (msg) => {
- if (msg.error) {
- reject();
- }
- resolve();
- });
- });
-
- // Copy config schemas to intellij
- const copyConfigSchemaChild = fork(
- path.join(__dirname, "generate-copy-config.js"),
- {
- stdio: "inherit",
- },
- );
- copyConfigSchemaChild.send({ payload: { operation: "copy" } });
-
- await new Promise((resolve, reject) => {
- copyConfigSchemaChild.on("message", (msg) => {
- if (msg.error) {
- reject();
- }
- resolve();
- });
- });
-}
-
function copyTokenizers() {
fs.copyFileSync(
path.join(__dirname, "../../../core/llm/llamaTokenizerWorkerPool.mjs"),
@@ -65,43 +22,6 @@ function copyTokenizers() {
console.log("[info] Copied llamaTokenizer");
}
-async function installNodeModules() {
- const installVscodeChild = fork(
- path.join(__dirname, "install-nodemodules.js"),
- {
- stdio: "inherit",
- },
- );
- installVscodeChild.send({ payload: { targetDir: "vscode" } });
-
- const installGuiChild = fork(path.join(__dirname, "install-nodemodules.js"), {
- stdio: "inherit",
- });
- installGuiChild.send({ payload: { targetDir: "gui" } });
-
- await Promise.all([
- new Promise((resolve, reject) => {
- installVscodeChild.on("message", (msg) => {
- if (msg.error) {
- reject();
- }
- resolve();
- });
- }),
- new Promise((resolve, reject) => {
- installGuiChild.on("message", (msg) => {
- if (msg.error) {
- reject();
- }
- resolve();
- });
- }),
- ]).catch((error) => {
- console.error(error);
- process.exit(1);
- });
-}
-
async function buildGui(isGhAction) {
// Make sure we are in the right directory
if (!process.cwd().endsWith("gui")) {
@@ -552,8 +472,6 @@ function writeBuildTimestamp() {
module.exports = {
continueDir,
- generateAndCopyConfigYamlSchema,
- installNodeModules,
buildGui,
copyOnnxRuntimeFromNodeModules,
copyTreeSitterWasms,
diff --git a/gui/package-lock.json b/gui/package-lock.json
index 020586519..a9d589b98 100644
--- a/gui/package-lock.json
+++ b/gui/package-lock.json
@@ -111,7 +111,7 @@
"@aws-sdk/credential-providers": "^3.778.0",
"@continuedev/config-types": "^1.0.13",
"@continuedev/config-yaml": "file:../packages/config-yaml",
- "@continuedev/fetch": "^1.0.10",
+ "@continuedev/fetch": "^1.0.13",
"@continuedev/llm-info": "^1.0.8",
"@continuedev/openai-adapters": "^1.0.25",
"@modelcontextprotocol/sdk": "^1.12.0",
From 3722b8c4f8a3896f2410897cb8b9874a4dec2bcb Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:51:55 +0530
Subject: [PATCH 13/20] refactor download node-sqlite to common
---
binary/utils/bundle-binary.js | 49 ++++++++-----------
.../scripts/install-copy-sqlite-esbuild.js | 33 +++++++------
2 files changed, 40 insertions(+), 42 deletions(-)
diff --git a/binary/utils/bundle-binary.js b/binary/utils/bundle-binary.js
index c218166cd..91d2f4593 100644
--- a/binary/utils/bundle-binary.js
+++ b/binary/utils/bundle-binary.js
@@ -8,41 +8,34 @@ const {
const { downloadRipgrep } = require("./ripgrep");
const { TARGET_TO_LANCEDB } = require("../utils/targets");
const fs = require("fs");
+const {
+ downloadSqlite,
+} = require("../../extensions/vscode/scripts/install-copy-sqlite-esbuild");
async function downloadNodeSqlite(target, targetDir) {
- return new Promise((resolve) => {
- const [currentPlatform, currentArch] = autodetectPlatformAndArch();
+ const [currentPlatform, currentArch] = autodetectPlatformAndArch();
- // Download and unzip prebuilt sqlite3 binary for the target
- console.log("[info] Downloading node-sqlite3");
+ // Download and unzip prebuilt sqlite3 binary for the target
+ console.log("[info] Downloading node-sqlite3");
- const downloadUrl =
- // node-sqlite3 doesn't have a pre-built binary for win32-arm64
- target === "win32-arm64"
- ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
- : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
- target
- }.tar.gz`;
+ await downloadSqlite(target, `${targetDir}/build.tar.gz`);
- execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
- execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
+ execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
- // Copy to build directory for testing
- try {
- const [platform, arch] = target.split("-");
- if (platform === currentPlatform && arch === currentArch) {
- fs.copyFileSync(
- `${targetDir}/build/Release/node_sqlite3.node`,
- `build/node_sqlite3.node`,
- );
- }
- } catch (error) {
- console.log("[warn] Could not copy node_sqlite to build");
- console.log(error);
+ // Copy to build directory for testing
+ try {
+ const [platform, arch] = target.split("-");
+ if (platform === currentPlatform && arch === currentArch) {
+ fs.copyFileSync(
+ `${targetDir}/build/Release/node_sqlite3.node`,
+ `build/node_sqlite3.node`,
+ );
}
- fs.unlinkSync(`${targetDir}/build.tar.gz`);
- resolve();
- });
+ } catch (error) {
+ console.log("[warn] Could not copy node_sqlite to build");
+ console.log(error);
+ }
+ fs.unlinkSync(`${targetDir}/build.tar.gz`);
}
/**
diff --git a/extensions/vscode/scripts/install-copy-sqlite-esbuild.js b/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
index 370876c7f..ad33b2eb6 100644
--- a/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
+++ b/extensions/vscode/scripts/install-copy-sqlite-esbuild.js
@@ -1,4 +1,4 @@
-const { fork, execSync } = require("child_process");
+const { fork } = require("child_process");
const fs = require("fs");
const https = require("https");
const path = require("path");
@@ -80,23 +80,27 @@ async function downloadFile(url, outputPath, maxRedirects = 5) {
});
}
+/**
+ *
+ * @param {string} target platform specific target
+ * @param {string} targetDir the directory to download into
+ */
+async function downloadSqlite(target, targetDir) {
+ const downloadUrl =
+ // node-sqlite3 doesn't have a pre-built binary for win32-arm64
+ target === "win32-arm64"
+ ? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
+ : `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
+ target
+ }.tar.gz`;
+ await downloadFile(downloadUrl, targetDir);
+}
+
async function installAndCopySqlite(target) {
// Replace the installed with pre-built
console.log("[info] Downloading pre-built sqlite3 binary");
rimrafSync("../../core/node_modules/sqlite3/build");
- const downloadUrl = {
- "darwin-arm64":
- "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-darwin-arm64.tar.gz",
- "linux-arm64":
- "https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v3-linux-arm64.tar.gz",
- // node-sqlite3 doesn't have a pre-built binary for win32-arm64
- "win32-arm64":
- "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz",
- }[target];
- await downloadFile(
- downloadUrl,
- "../../core/node_modules/sqlite3/build.tar.gz",
- );
+ await downloadSqlite(target, "../../core/node_modules/sqlite3/build.tar.gz");
execCmdSync("cd ../../core/node_modules/sqlite3 && tar -xvzf build.tar.gz");
fs.unlinkSync("../../core/node_modules/sqlite3/build.tar.gz");
}
@@ -179,6 +183,7 @@ async function copyEsbuild(target) {
}
module.exports = {
+ downloadSqlite,
copySqlite,
copyEsbuild,
};
From e8e8821134ef15043b92de83a31fa203640f6d70 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:58:45 +0530
Subject: [PATCH 14/20] move install-copy-nodemodule to vscode scripts
---
binary/build.js | 6 ++++--
extensions/vscode/package-lock.json | 4 ++--
.../vscode/scripts/install-copy-nodemodule.js | 12 +++++++-----
extensions/vscode/scripts/prepackage.js | 6 +++---
package-lock.json | 11 -----------
package.json | 1 -
6 files changed, 16 insertions(+), 24 deletions(-)
rename scripts/util/copy-nodemodule.js => extensions/vscode/scripts/install-copy-nodemodule.js (92%)
diff --git a/binary/build.js b/binary/build.js
index aa5fa5ffd..56c27b2bd 100644
--- a/binary/build.js
+++ b/binary/build.js
@@ -6,7 +6,9 @@ const { rimrafSync } = require("rimraf");
const { validateFilesPresent } = require("../scripts/util");
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
const { fork } = require("child_process");
-const { copyNodeModule } = require("../scripts/util/copy-nodemodule");
+const {
+ installAndCopyNodeModules,
+} = require("../extensions/vscode/scripts/install-copy-nodemodule");
const bin = path.join(__dirname, "bin");
const out = path.join(__dirname, "out");
@@ -104,7 +106,7 @@ async function buildWithEsbuild() {
}
console.log(`[info] Downloading for ${target}...`);
copyLanceDBPromises.push(
- copyNodeModule(TARGET_TO_LANCEDB[target], "@lancedb"),
+ installAndCopyNodeModules(TARGET_TO_LANCEDB[target], "@lancedb"),
);
}
await Promise.all(copyLanceDBPromises).catch(() => {
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index 38efaa08a..349212a6a 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "continue",
- "version": "1.1.44",
+ "version": "1.1.45",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "continue",
- "version": "1.1.44",
+ "version": "1.1.45",
"license": "Apache-2.0",
"dependencies": {
"@continuedev/config-types": "^1.0.14",
diff --git a/scripts/util/copy-nodemodule.js b/extensions/vscode/scripts/install-copy-nodemodule.js
similarity index 92%
rename from scripts/util/copy-nodemodule.js
rename to extensions/vscode/scripts/install-copy-nodemodule.js
index 361e2d891..a20d2dee4 100644
--- a/scripts/util/copy-nodemodule.js
+++ b/extensions/vscode/scripts/install-copy-nodemodule.js
@@ -1,12 +1,14 @@
/**
- * @file Copy lancedb to the current directory. It is intended to run as a child process.
+ * @file Copy lancedb to the current directory. It is also intended to run as a child process.
*/
+const { fork } = require("child_process");
const fs = require("fs");
const path = require("path");
+
const ncp = require("ncp").ncp;
-const { execCmdSync } = require(".");
-const { fork } = require("child_process");
+
+const { execCmdSync } = require("../../../scripts/util");
async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
console.log(`Copying ${packageName} to ${toCopy}`);
@@ -86,7 +88,7 @@ process.on("message", (msg) => {
* @param {string} packageName the module to install and copy
* @param {string} toCopy directory to copy into inside node modules
*/
-async function copyNodeModule(packageName, toCopy) {
+async function installAndCopyNodeModules(packageName, toCopy) {
const child = fork(__filename, { stdio: "inherit", cwd: process.cwd() });
child.send({
payload: {
@@ -106,5 +108,5 @@ async function copyNodeModule(packageName, toCopy) {
}
module.exports = {
- copyNodeModule,
+ installAndCopyNodeModules,
};
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 622393a19..57fa5a52d 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -4,7 +4,6 @@ const path = require("path");
const ncp = require("ncp").ncp;
const { rimrafSync } = require("rimraf");
-const { copyNodeModule } = require("../../../scripts/util/copy-nodemodule");
const {
validateFilesPresent,
execCmdSync,
@@ -12,6 +11,7 @@ const {
} = require("../../../scripts/util/index");
const { generateAndCopyConfigYamlSchema } = require("./generate-copy-config");
+const { installAndCopyNodeModules } = require("./install-copy-nodemodule");
const { copySqlite, copyEsbuild } = require("./install-copy-sqlite-esbuild");
const { installNodeModules } = require("./install-nodemodules");
const { writeBuildTimestamp, continueDir } = require("./utils");
@@ -280,12 +280,12 @@ void (async () => {
await Promise.all([
copyEsbuild(target),
copySqlite(target),
- copyNodeModule(packageToInstall, "@lancedb"),
+ installAndCopyNodeModules(packageToInstall, "@lancedb"),
]);
} else {
// Download esbuild from npm in tmp and copy over
console.log("[info] npm installing esbuild binary");
- await copyNodeModule("esbuild@0.17.19", "@esbuild");
+ await installAndCopyNodeModules("esbuild@0.17.19", "@esbuild");
}
console.log("[info] Copying sqlite node binding from core");
diff --git a/package-lock.json b/package-lock.json
index 43530562e..06f811afa 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,7 +8,6 @@
"@typescript-eslint/parser": "^7.8.0",
"concurrently": "^9.1.2",
"eslint-plugin-import": "^2.29.1",
- "ncp": "^2.0.0",
"prettier": "^3.3.3",
"prettier-plugin-tailwindcss": "^0.6.8",
"typescript": "^5.6.3"
@@ -2234,16 +2233,6 @@
"dev": true,
"peer": true
},
- "node_modules/ncp": {
- "version": "2.0.0",
- "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
- "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==",
- "dev": true,
- "license": "MIT",
- "bin": {
- "ncp": "bin/ncp"
- }
- },
"node_modules/object-inspect": {
"version": "1.13.2",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz",
diff --git a/package.json b/package.json
index 9100ad0cd..79e522af1 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,6 @@
"@typescript-eslint/parser": "^7.8.0",
"concurrently": "^9.1.2",
"eslint-plugin-import": "^2.29.1",
- "ncp": "^2.0.0",
"prettier": "^3.3.3",
"prettier-plugin-tailwindcss": "^0.6.8",
"typescript": "^5.6.3"
From 1d4a31af2243297e1582a6ae5ed96f57a4212bfe Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:59:39 +0530
Subject: [PATCH 15/20] rename install-copy-sqlite-esbuild to
download-copy-sqlite-esbuild
---
binary/utils/bundle-binary.js | 2 +-
...l-copy-sqlite-esbuild.js => download-copy-sqlite-esbuild.js} | 0
extensions/vscode/scripts/prepackage.js | 2 +-
3 files changed, 2 insertions(+), 2 deletions(-)
rename extensions/vscode/scripts/{install-copy-sqlite-esbuild.js => download-copy-sqlite-esbuild.js} (100%)
diff --git a/binary/utils/bundle-binary.js b/binary/utils/bundle-binary.js
index 91d2f4593..50f34f647 100644
--- a/binary/utils/bundle-binary.js
+++ b/binary/utils/bundle-binary.js
@@ -10,7 +10,7 @@ const { TARGET_TO_LANCEDB } = require("../utils/targets");
const fs = require("fs");
const {
downloadSqlite,
-} = require("../../extensions/vscode/scripts/install-copy-sqlite-esbuild");
+} = require("../../extensions/vscode/scripts/download-copy-sqlite-esbuild");
async function downloadNodeSqlite(target, targetDir) {
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
diff --git a/extensions/vscode/scripts/install-copy-sqlite-esbuild.js b/extensions/vscode/scripts/download-copy-sqlite-esbuild.js
similarity index 100%
rename from extensions/vscode/scripts/install-copy-sqlite-esbuild.js
rename to extensions/vscode/scripts/download-copy-sqlite-esbuild.js
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 57fa5a52d..7649e0919 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -10,9 +10,9 @@ const {
autodetectPlatformAndArch,
} = require("../../../scripts/util/index");
+const { copySqlite, copyEsbuild } = require("./download-copy-sqlite-esbuild");
const { generateAndCopyConfigYamlSchema } = require("./generate-copy-config");
const { installAndCopyNodeModules } = require("./install-copy-nodemodule");
-const { copySqlite, copyEsbuild } = require("./install-copy-sqlite-esbuild");
const { installNodeModules } = require("./install-nodemodules");
const { writeBuildTimestamp, continueDir } = require("./utils");
From a363e34f753bbf69b22d33e9c4a03f937b3fa038 Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 23:03:20 +0530
Subject: [PATCH 16/20] refactor bundle binary to modular function
---
binary/build.js | 18 ++----------------
binary/utils/bundle-binary.js | 28 +++++++++++++++++++++++++++-
2 files changed, 29 insertions(+), 17 deletions(-)
diff --git a/binary/build.js b/binary/build.js
index 56c27b2bd..c2431333c 100644
--- a/binary/build.js
+++ b/binary/build.js
@@ -9,6 +9,7 @@ const { fork } = require("child_process");
const {
installAndCopyNodeModules,
} = require("../extensions/vscode/scripts/install-copy-nodemodule");
+const { bundleBinary } = require("./utils/bundle-binary");
const bin = path.join(__dirname, "bin");
const out = path.join(__dirname, "out");
@@ -178,22 +179,7 @@ async function buildWithEsbuild() {
const buildBinaryPromises = [];
console.log("[info] Building binaries with pkg...");
for (const target of targets) {
- const child = fork("./utils/bundle-binary.js", { stdio: "inherit" });
- child.send({
- payload: {
- target,
- },
- });
- buildBinaryPromises.push(
- new Promise((resolve, reject) => {
- child.on("message", (msg) => {
- if (msg.error) {
- reject();
- }
- resolve();
- });
- }),
- );
+ buildBinaryPromises.push(bundleBinary(target));
}
await Promise.all(buildBinaryPromises).catch(() => {
console.error("[error] Failed to build binaries");
diff --git a/binary/utils/bundle-binary.js b/binary/utils/bundle-binary.js
index 50f34f647..0a778f5b0 100644
--- a/binary/utils/bundle-binary.js
+++ b/binary/utils/bundle-binary.js
@@ -1,6 +1,7 @@
/**
- * @file Builds the binary for the specified target. It is intended to run as a child process.
+ * @file Builds the binary for the specified target. It is also intended to run as a child process.
*/
+
const {
execCmdSync,
autodetectPlatformAndArch,
@@ -11,6 +12,7 @@ const fs = require("fs");
const {
downloadSqlite,
} = require("../../extensions/vscode/scripts/download-copy-sqlite-esbuild");
+const { fork } = require("child_process");
async function downloadNodeSqlite(target, targetDir) {
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
@@ -74,3 +76,27 @@ process.on("message", (msg) => {
process.send({ error: true });
});
});
+
+/**
+ * @param {string} target the platform to bundle for
+ */
+async function bundleBinary(target) {
+ const child = fork(__filename, { stdio: "inherit" });
+ child.send({
+ payload: {
+ target,
+ },
+ });
+ return new Promise((resolve, reject) => {
+ child.on("message", (msg) => {
+ if (msg.error) {
+ reject();
+ }
+ resolve();
+ });
+ });
+}
+
+module.exports = {
+ bundleBinary,
+};
From 9c087bb8b8dd80fe55dd2175b0760b4143ed298f Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 23:04:08 +0530
Subject: [PATCH 17/20] revert package-lock to main
---
binary/package-lock.json | 5 ++---
extensions/vscode/package-lock.json | 4 ++--
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/binary/package-lock.json b/binary/package-lock.json
index 2d84672f0..abb11ce8e 100644
--- a/binary/package-lock.json
+++ b/binary/package-lock.json
@@ -52,7 +52,7 @@
"@aws-sdk/credential-providers": "^3.778.0",
"@continuedev/config-types": "^1.0.13",
"@continuedev/config-yaml": "file:../packages/config-yaml",
- "@continuedev/fetch": "^1.0.13",
+ "@continuedev/fetch": "^1.0.10",
"@continuedev/llm-info": "^1.0.8",
"@continuedev/openai-adapters": "^1.0.25",
"@modelcontextprotocol/sdk": "^1.12.0",
@@ -144,8 +144,7 @@
"onnxruntime-common": "1.14.0",
"onnxruntime-web": "1.14.0",
"ts-jest": "^29.1.1",
- "typescript": "^5.6.3",
- "vitest": "^3.1.4"
+ "typescript": "^5.6.3"
},
"engines": {
"node": ">=20.19.0"
diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json
index 349212a6a..38efaa08a 100644
--- a/extensions/vscode/package-lock.json
+++ b/extensions/vscode/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "continue",
- "version": "1.1.45",
+ "version": "1.1.44",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "continue",
- "version": "1.1.45",
+ "version": "1.1.44",
"license": "Apache-2.0",
"dependencies": {
"@continuedev/config-types": "^1.0.14",
From 1414aec53ba347d36c1b978b4fdd04d77d0657bf Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Fri, 6 Jun 2025 23:08:21 +0530
Subject: [PATCH 18/20] rename install nodemodules to npm-install
---
.../vscode/scripts/generate-copy-config.js | 2 +-
.../{install-nodemodules.js => npm-install.js} | 17 +++++++----------
extensions/vscode/scripts/prepackage.js | 4 ++--
3 files changed, 10 insertions(+), 13 deletions(-)
rename extensions/vscode/scripts/{install-nodemodules.js => npm-install.js} (82%)
diff --git a/extensions/vscode/scripts/generate-copy-config.js b/extensions/vscode/scripts/generate-copy-config.js
index 73fe4602d..21a68d999 100644
--- a/extensions/vscode/scripts/generate-copy-config.js
+++ b/extensions/vscode/scripts/generate-copy-config.js
@@ -1,5 +1,5 @@
/**
- * @file Generate config.yaml file from template. Intended to run as a child process.
+ * @file Generate config.yaml file from template. Also intended to run as a child process.
*/
const { fork } = require("child_process");
diff --git a/extensions/vscode/scripts/install-nodemodules.js b/extensions/vscode/scripts/npm-install.js
similarity index 82%
rename from extensions/vscode/scripts/install-nodemodules.js
rename to extensions/vscode/scripts/npm-install.js
index f23de3b3b..feffdcf26 100644
--- a/extensions/vscode/scripts/install-nodemodules.js
+++ b/extensions/vscode/scripts/npm-install.js
@@ -1,5 +1,5 @@
/**
- * @file Install node modules for the VS Code extension. This is intended to run as a child process.
+ * @file Install node modules for the VS Code extension and gui. This is also intended to run as a child process.
*/
const { fork } = require("child_process");
@@ -40,16 +40,13 @@ process.on("message", (msg) => {
}
});
-async function installNodeModules() {
- const installVscodeChild = fork(
- path.join(__dirname, "install-nodemodules.js"),
- {
- stdio: "inherit",
- },
- );
+async function npmInstall() {
+ const installVscodeChild = fork(__filename, {
+ stdio: "inherit",
+ });
installVscodeChild.send({ payload: { targetDir: "vscode" } });
- const installGuiChild = fork(path.join(__dirname, "install-nodemodules.js"), {
+ const installGuiChild = fork(__filename, {
stdio: "inherit",
});
installGuiChild.send({ payload: { targetDir: "gui" } });
@@ -78,5 +75,5 @@ async function installNodeModules() {
}
module.exports = {
- installNodeModules,
+ npmInstall,
};
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 7649e0919..9fb0d43a4 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -13,7 +13,7 @@ const {
const { copySqlite, copyEsbuild } = require("./download-copy-sqlite-esbuild");
const { generateAndCopyConfigYamlSchema } = require("./generate-copy-config");
const { installAndCopyNodeModules } = require("./install-copy-nodemodule");
-const { installNodeModules } = require("./install-nodemodules");
+const { npmInstall } = require("./npm-install");
const { writeBuildTimestamp, continueDir } = require("./utils");
// Clear folders that will be packaged to ensure clean slate
@@ -70,7 +70,7 @@ void (async () => {
// Make sure we have an initial timestamp file
writeBuildTimestamp();
- await Promise.all([generateAndCopyConfigYamlSchema(), installNodeModules()]);
+ await Promise.all([generateAndCopyConfigYamlSchema(), npmInstall()]);
process.chdir(path.join(continueDir, "gui"));
From 01930c926d563b1241ed70b86b1887ed074b08cd Mon Sep 17 00:00:00 2001
From: uinstinct <61635505+uinstinct@users.noreply.github.com>
Date: Thu, 19 Jun 2025 16:41:16 +0530
Subject: [PATCH 19/20] skip installations and downloads on extension run
---
.vscode/tasks.json | 3 ++
extensions/vscode/scripts/prepackage.js | 50 ++++++++++++++-----------
scripts/install-dependencies.sh | 2 +-
3 files changed, 32 insertions(+), 23 deletions(-)
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
index e40d9d054..d3a4a623d 100644
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@@ -103,6 +103,9 @@
"clear": true
},
"options": {
+ "env": {
+ "SKIP_INSTALLS": "true"
+ },
"cwd": "${workspaceFolder}/extensions/vscode"
}
},
diff --git a/extensions/vscode/scripts/prepackage.js b/extensions/vscode/scripts/prepackage.js
index 9fb0d43a4..599b06062 100644
--- a/extensions/vscode/scripts/prepackage.js
+++ b/extensions/vscode/scripts/prepackage.js
@@ -27,6 +27,8 @@ if (!fs.existsSync(guiDist)) {
fs.mkdirSync(guiDist, { recursive: true });
}
+const skipInstalls = process.env.SKIP_INSTALLS === "true";
+
// Get the target to package for
let target = undefined;
const args = process.argv;
@@ -70,7 +72,9 @@ void (async () => {
// Make sure we have an initial timestamp file
writeBuildTimestamp();
- await Promise.all([generateAndCopyConfigYamlSchema(), npmInstall()]);
+ if (!skipInstalls) {
+ await Promise.all([generateAndCopyConfigYamlSchema(), npmInstall()]);
+ }
process.chdir(path.join(continueDir, "gui"));
@@ -264,28 +268,30 @@ void (async () => {
);
});
- // GitHub Actions doesn't support ARM, so we need to download pre-saved binaries
- // 02/07/25 - the above comment is out of date, there is now support for ARM runners on GitHub Actions
- if (isArmTarget) {
- // lancedb binary
- const packageToInstall = {
- "darwin-arm64": "@lancedb/vectordb-darwin-arm64",
- "linux-arm64": "@lancedb/vectordb-linux-arm64-gnu",
- "win32-arm64": "@lancedb/vectordb-win32-arm64-msvc",
- }[target];
- console.log(
- "[info] Downloading pre-built lancedb binary: " + packageToInstall,
- );
+ if (!skipInstalls) {
+ // GitHub Actions doesn't support ARM, so we need to download pre-saved binaries
+ // 02/07/25 - the above comment is out of date, there is now support for ARM runners on GitHub Actions
+ if (isArmTarget) {
+ // lancedb binary
+ const packageToInstall = {
+ "darwin-arm64": "@lancedb/vectordb-darwin-arm64",
+ "linux-arm64": "@lancedb/vectordb-linux-arm64-gnu",
+ "win32-arm64": "@lancedb/vectordb-win32-arm64-msvc",
+ }[target];
+ console.log(
+ "[info] Downloading pre-built lancedb binary: " + packageToInstall,
+ );
- await Promise.all([
- copyEsbuild(target),
- copySqlite(target),
- installAndCopyNodeModules(packageToInstall, "@lancedb"),
- ]);
- } else {
- // Download esbuild from npm in tmp and copy over
- console.log("[info] npm installing esbuild binary");
- await installAndCopyNodeModules("esbuild@0.17.19", "@esbuild");
+ await Promise.all([
+ copyEsbuild(target),
+ copySqlite(target),
+ installAndCopyNodeModules(packageToInstall, "@lancedb"),
+ ]);
+ } else {
+ // Download esbuild from npm in tmp and copy over
+ console.log("[info] npm installing esbuild binary");
+ await installAndCopyNodeModules("esbuild@0.17.19", "@esbuild");
+ }
}
console.log("[info] Copying sqlite node binding from core");
diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh
index d92fc109d..092028b37 100755
--- a/scripts/install-dependencies.sh
+++ b/scripts/install-dependencies.sh
@@ -57,7 +57,7 @@ pushd extensions/vscode
# This does way too many things inline but is the common denominator between many of the scripts
npm install
npm link @continuedev/core
-npm run prepackage
+# npm run prepackage # not required since npm run package has prescript of prepackage
npm run package
popd
From a6dbe946eab3591ed143fb5a89cad8d7be6a243a Mon Sep 17 00:00:00 2001
From: Nate
Date: Thu, 26 Jun 2025 20:54:15 -0700
Subject: [PATCH 20/20] address feedback from recurse
---
binary/utils/bundle-binary.js | 3 ++-
extensions/vscode/scripts/download-copy-sqlite-esbuild.js | 6 ++++--
extensions/vscode/scripts/esbuild.js | 3 ++-
extensions/vscode/scripts/install-copy-nodemodule.js | 3 ++-
4 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/binary/utils/bundle-binary.js b/binary/utils/bundle-binary.js
index 0a778f5b0..f0cde0d25 100644
--- a/binary/utils/bundle-binary.js
+++ b/binary/utils/bundle-binary.js
@@ -91,8 +91,9 @@ async function bundleBinary(target) {
child.on("message", (msg) => {
if (msg.error) {
reject();
+ } else {
+ resolve();
}
- resolve();
});
});
}
diff --git a/extensions/vscode/scripts/download-copy-sqlite-esbuild.js b/extensions/vscode/scripts/download-copy-sqlite-esbuild.js
index ad33b2eb6..ea6f8ebd9 100644
--- a/extensions/vscode/scripts/download-copy-sqlite-esbuild.js
+++ b/extensions/vscode/scripts/download-copy-sqlite-esbuild.js
@@ -154,8 +154,9 @@ async function copySqlite(target) {
child.on("message", (msg) => {
if (msg.error) {
reject();
+ } else {
+ resolve();
}
- resolve();
});
});
}
@@ -176,8 +177,9 @@ async function copyEsbuild(target) {
child.on("message", (msg) => {
if (msg.error) {
reject();
+ } else {
+ resolve();
}
- resolve();
});
});
}
diff --git a/extensions/vscode/scripts/esbuild.js b/extensions/vscode/scripts/esbuild.js
index 9382e2b7a..e800cdfc6 100644
--- a/extensions/vscode/scripts/esbuild.js
+++ b/extensions/vscode/scripts/esbuild.js
@@ -1,4 +1,5 @@
const fs = require("fs");
+
const { writeBuildTimestamp } = require("./utils");
const esbuild = require("esbuild");
@@ -49,7 +50,7 @@ const esbuildConfig = {
],
};
-(async () => {
+void (async () => {
// Create .buildTimestamp.js before starting the first build
writeBuildTimestamp();
// Bundles the extension into one file
diff --git a/extensions/vscode/scripts/install-copy-nodemodule.js b/extensions/vscode/scripts/install-copy-nodemodule.js
index a20d2dee4..1ce65327a 100644
--- a/extensions/vscode/scripts/install-copy-nodemodule.js
+++ b/extensions/vscode/scripts/install-copy-nodemodule.js
@@ -101,8 +101,9 @@ async function installAndCopyNodeModules(packageName, toCopy) {
child.on("message", (msg) => {
if (msg.error) {
reject();
+ } else {
+ resolve();
}
- resolve();
});
});
}