Merge pull request #6022 from uinstinct/speedup-builds
chore: speedup builds for binary and prepackage
This commit is contained in:
Nate Sesti
GitHub
170
binary/build.js
170
binary/build.js
@@ -3,13 +3,13 @@ const fs = require("fs");
|
||||
const path = require("path");
|
||||
const ncp = require("ncp").ncp;
|
||||
const { rimrafSync } = require("rimraf");
|
||||
const {
|
||||
validateFilesPresent,
|
||||
execCmdSync,
|
||||
autodetectPlatformAndArch,
|
||||
} = require("../scripts/util");
|
||||
const { downloadRipgrep } = require("./utils/ripgrep");
|
||||
const { validateFilesPresent } = require("../scripts/util");
|
||||
const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
|
||||
const { fork } = require("child_process");
|
||||
const {
|
||||
installAndCopyNodeModules,
|
||||
} = require("../extensions/vscode/scripts/install-copy-nodemodule");
|
||||
const { bundleBinary } = require("./utils/bundle-binary");
|
||||
|
||||
const bin = path.join(__dirname, "bin");
|
||||
const out = path.join(__dirname, "out");
|
||||
@@ -29,8 +29,6 @@ function cleanSlate() {
|
||||
const esbuildOutputFile = "out/index.js";
|
||||
let targets = [...ALL_TARGETS];
|
||||
|
||||
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
|
||||
|
||||
const assetBackups = [
|
||||
"node_modules/win-ca/lib/crypt32-ia32.node.bak",
|
||||
"node_modules/win-ca/lib/crypt32-x64.node.bak",
|
||||
@@ -78,85 +76,6 @@ async function buildWithEsbuild() {
|
||||
});
|
||||
}
|
||||
|
||||
async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
|
||||
console.log(`Copying ${packageName} to ${toCopy}`);
|
||||
// This is a way to install only one package without npm trying to install all the dependencies
|
||||
// Create a temporary directory for installing the package
|
||||
const adjustedName = packageName.replace(/@/g, "").replace("/", "-");
|
||||
const tempDir = path.join(
|
||||
__dirname,
|
||||
"tmp",
|
||||
`continue-node_modules-${adjustedName}`,
|
||||
);
|
||||
const currentDir = process.cwd();
|
||||
|
||||
// // Remove the dir we will be copying to
|
||||
// rimrafSync(`node_modules/${toCopy}`);
|
||||
|
||||
// // Ensure the temporary directory exists
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
// Move to the temporary directory
|
||||
process.chdir(tempDir);
|
||||
|
||||
// Initialize a new package.json and install the package
|
||||
execCmdSync(`npm init -y && npm i -f ${packageName} --no-save`);
|
||||
|
||||
console.log(
|
||||
`Contents of: ${packageName}`,
|
||||
fs.readdirSync(path.join(tempDir, "node_modules", toCopy)),
|
||||
);
|
||||
|
||||
// Without this it seems the file isn't completely written to disk
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
// Copy the installed package back to the current directory
|
||||
await new Promise((resolve, reject) => {
|
||||
ncp(
|
||||
path.join(tempDir, "node_modules", toCopy),
|
||||
path.join(currentDir, "node_modules", toCopy),
|
||||
{ dereference: true },
|
||||
(error) => {
|
||||
if (error) {
|
||||
console.error(
|
||||
`[error] Error copying ${packageName} package`,
|
||||
error,
|
||||
);
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
// Clean up the temporary directory
|
||||
// rimrafSync(tempDir);
|
||||
|
||||
// Return to the original directory
|
||||
process.chdir(currentDir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and installs ripgrep binaries for the specified target
|
||||
*
|
||||
* @param {string} target - Target platform-arch (e.g., 'darwin-x64')
|
||||
* @param {string} targetDir - Directory to install ripgrep to
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function downloadRipgrepForTarget(target, targetDir) {
|
||||
console.log(`[info] Downloading ripgrep for ${target}...`);
|
||||
try {
|
||||
await downloadRipgrep(target, targetDir);
|
||||
console.log(`[info] Successfully installed ripgrep for ${target}`);
|
||||
} catch (error) {
|
||||
console.error(`[error] Failed to download ripgrep for ${target}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
if (esbuildOnly) {
|
||||
await buildWithEsbuild();
|
||||
@@ -181,16 +100,21 @@ async function downloadRipgrepForTarget(target, targetDir) {
|
||||
),
|
||||
);
|
||||
|
||||
console.log("[info] Downloading prebuilt lancedb...");
|
||||
const copyLanceDBPromises = [];
|
||||
for (const target of targets) {
|
||||
if (TARGET_TO_LANCEDB[target]) {
|
||||
console.log(`[info] Downloading for ${target}...`);
|
||||
await installNodeModuleInTempDirAndCopyToCurrent(
|
||||
TARGET_TO_LANCEDB[target],
|
||||
"@lancedb",
|
||||
);
|
||||
if (!TARGET_TO_LANCEDB[target]) {
|
||||
continue;
|
||||
}
|
||||
console.log(`[info] Downloading for ${target}...`);
|
||||
copyLanceDBPromises.push(
|
||||
installAndCopyNodeModules(TARGET_TO_LANCEDB[target], "@lancedb"),
|
||||
);
|
||||
}
|
||||
await Promise.all(copyLanceDBPromises).catch(() => {
|
||||
console.error("[error] Failed to copy LanceDB");
|
||||
process.exit(1);
|
||||
});
|
||||
console.log("[info] Copied all LanceDB");
|
||||
|
||||
// tree-sitter-wasm
|
||||
const treeSitterWasmsDir = path.join(out, "tree-sitter-wasms");
|
||||
@@ -252,59 +176,16 @@ async function downloadRipgrepForTarget(target, targetDir) {
|
||||
"out/llamaTokenizerWorkerPool.mjs",
|
||||
);
|
||||
|
||||
const buildBinaryPromises = [];
|
||||
console.log("[info] Building binaries with pkg...");
|
||||
for (const target of targets) {
|
||||
const targetDir = `bin/${target}`;
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
console.log(`[info] Building ${target}...`);
|
||||
execCmdSync(
|
||||
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
|
||||
);
|
||||
|
||||
// Download and unzip prebuilt sqlite3 binary for the target
|
||||
console.log("[info] Downloading node-sqlite3");
|
||||
|
||||
const downloadUrl =
|
||||
// node-sqlite3 doesn't have a pre-built binary for win32-arm64
|
||||
target === "win32-arm64"
|
||||
? "https://continue-server-binaries.s3.us-west-1.amazonaws.com/win32-arm64/node_sqlite3.tar.gz"
|
||||
: `https://github.com/TryGhost/node-sqlite3/releases/download/v5.1.7/sqlite3-v5.1.7-napi-v6-${
|
||||
target
|
||||
}.tar.gz`;
|
||||
|
||||
execCmdSync(`curl -L -o ${targetDir}/build.tar.gz ${downloadUrl}`);
|
||||
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
|
||||
|
||||
// Copy to build directory for testing
|
||||
try {
|
||||
const [platform, arch] = target.split("-");
|
||||
if (platform === currentPlatform && arch === currentArch) {
|
||||
fs.copyFileSync(
|
||||
`${targetDir}/build/Release/node_sqlite3.node`,
|
||||
`build/node_sqlite3.node`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("[warn] Could not copy node_sqlite to build");
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
fs.unlinkSync(`${targetDir}/build.tar.gz`);
|
||||
|
||||
// copy @lancedb to bin folders
|
||||
console.log("[info] Copying @lancedb files to bin");
|
||||
fs.copyFileSync(
|
||||
`node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
|
||||
`${targetDir}/index.node`,
|
||||
);
|
||||
|
||||
// Download and install ripgrep for the target
|
||||
await downloadRipgrepForTarget(target, targetDir);
|
||||
|
||||
// Informs the `continue-binary` of where to look for node_sqlite3.node
|
||||
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
|
||||
fs.writeFileSync(`${targetDir}/package.json`, "");
|
||||
buildBinaryPromises.push(bundleBinary(target));
|
||||
}
|
||||
await Promise.all(buildBinaryPromises).catch(() => {
|
||||
console.error("[error] Failed to build binaries");
|
||||
process.exit(1);
|
||||
});
|
||||
console.log("[info] All binaries built");
|
||||
|
||||
// Cleanup - this is needed when running locally
|
||||
fs.rmSync("out/package.json");
|
||||
@@ -331,4 +212,5 @@ async function downloadRipgrepForTarget(target, targetDir) {
|
||||
validateFilesPresent(pathsToVerify);
|
||||
|
||||
console.log("[info] Done!");
|
||||
process.exit(0);
|
||||
})();
|
||||
|
||||
103
binary/utils/bundle-binary.js
Normal file
103
binary/utils/bundle-binary.js
Normal file
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* @file Builds the binary for the specified target. It is also intended to run as a child process.
|
||||
*/
|
||||
|
||||
const {
|
||||
execCmdSync,
|
||||
autodetectPlatformAndArch,
|
||||
} = require("../../scripts/util");
|
||||
const { downloadRipgrep } = require("./ripgrep");
|
||||
const { TARGET_TO_LANCEDB } = require("../utils/targets");
|
||||
const fs = require("fs");
|
||||
const {
|
||||
downloadSqlite,
|
||||
} = require("../../extensions/vscode/scripts/download-copy-sqlite-esbuild");
|
||||
const { fork } = require("child_process");
|
||||
|
||||
async function downloadNodeSqlite(target, targetDir) {
|
||||
const [currentPlatform, currentArch] = autodetectPlatformAndArch();
|
||||
|
||||
// Download and unzip prebuilt sqlite3 binary for the target
|
||||
console.log("[info] Downloading node-sqlite3");
|
||||
|
||||
await downloadSqlite(target, `${targetDir}/build.tar.gz`);
|
||||
|
||||
execCmdSync(`cd ${targetDir} && tar -xvzf build.tar.gz`);
|
||||
|
||||
// Copy to build directory for testing
|
||||
try {
|
||||
const [platform, arch] = target.split("-");
|
||||
if (platform === currentPlatform && arch === currentArch) {
|
||||
fs.copyFileSync(
|
||||
`${targetDir}/build/Release/node_sqlite3.node`,
|
||||
`build/node_sqlite3.node`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("[warn] Could not copy node_sqlite to build");
|
||||
console.log(error);
|
||||
}
|
||||
fs.unlinkSync(`${targetDir}/build.tar.gz`);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} target the platform to build for
|
||||
*/
|
||||
async function bundleForBinary(target) {
|
||||
const targetDir = `bin/${target}`;
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
console.log(`[info] Building ${target}...`);
|
||||
execCmdSync(
|
||||
`npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
|
||||
);
|
||||
|
||||
// copy @lancedb to bin folders
|
||||
console.log("[info] Copying @lancedb files to bin");
|
||||
fs.copyFileSync(
|
||||
`node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
|
||||
`${targetDir}/index.node`,
|
||||
);
|
||||
|
||||
const downloadPromises = [];
|
||||
downloadPromises.push(downloadRipgrep(target, targetDir));
|
||||
downloadPromises.push(downloadNodeSqlite(target, targetDir));
|
||||
await Promise.all(downloadPromises);
|
||||
|
||||
// Informs the `continue-binary` of where to look for node_sqlite3.node
|
||||
// https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
|
||||
fs.writeFileSync(`${targetDir}/package.json`, "");
|
||||
}
|
||||
|
||||
process.on("message", (msg) => {
|
||||
bundleForBinary(msg.payload.target)
|
||||
.then(() => process.send({ done: true }))
|
||||
.catch((error) => {
|
||||
console.error(error); // show the error in the parent process
|
||||
process.send({ error: true });
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {string} target the platform to bundle for
|
||||
*/
|
||||
async function bundleBinary(target) {
|
||||
const child = fork(__filename, { stdio: "inherit" });
|
||||
child.send({
|
||||
payload: {
|
||||
target,
|
||||
},
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on("message", (msg) => {
|
||||
if (msg.error) {
|
||||
reject();
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
bundleBinary,
|
||||
};
|
||||
Reference in New Issue
Block a user