From c3fcc15cfa5db621d1ac808a1885748377edd7da Mon Sep 17 00:00:00 2001 From: dive2tech Date: Mon, 26 Jan 2026 09:11:01 -0500 Subject: [PATCH] feat: improve text/binary file detection in hash.js (#17543) * Improve text/binary file detection in hash.js Replace hardcoded text: true with proper detection logic that: - Checks for null bytes to identify binary files - Validates UTF-8 encoding with round-trip verification - Handles binary files by encoding them as base64 This fixes the TODO comment and makes the script more robust. * tweak * snake_case --------- Co-authored-by: Gittensor Miner Co-authored-by: Rich Harris --- playgrounds/sandbox/scripts/hash.js | 32 +++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/playgrounds/sandbox/scripts/hash.js b/playgrounds/sandbox/scripts/hash.js index e70dfc471a..fdfe2d1aa7 100644 --- a/playgrounds/sandbox/scripts/hash.js +++ b/playgrounds/sandbox/scripts/hash.js @@ -1,16 +1,44 @@ import fs from 'node:fs'; +/** + * Detects if a file is text or binary by checking for null bytes + * and validating UTF-8 encoding + * @param {string} filepath - Path to the file + * @returns {boolean} - true if file is text, false if binary + */ +function is_text_file(filepath) { + const buffer = fs.readFileSync(filepath); + // Check for null bytes which indicate binary files + for (let i = 0; i < buffer.length; i++) { + if (buffer[i] === 0) { + return false; + } + } + // Validate UTF-8 encoding + try { + const text = buffer.toString('utf-8'); + // Verify round-trip encoding to ensure valid UTF-8 + const encoded = Buffer.from(text, 'utf-8'); + return buffer.equals(encoded); + } catch { + return false; + } +} + const files = []; for (const basename of fs.readdirSync('src')) { if (fs.statSync(`src/${basename}`).isDirectory()) continue; + const filepath = `src/${basename}`; + const text = is_text_file(filepath); + files.push({ type: 'file', name: basename, basename, - contents: fs.readFileSync(`src/${basename}`, 'utf-8'), - text: true // TODO might not be + contents: fs.readFileSync(filepath, text ? 'utf-8' : 'base64'), + text }); }