From 927789bcb88d54f934debcb04f29b18d8eb01eb6 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Sun, 11 Jan 2026 19:04:49 -0500 Subject: [PATCH 1/7] feat: Mark down plugins script --- .gitignore | 3 +- package.json | 1 + scripts/mark-plugin-sites.js | 214 +++++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 scripts/mark-plugin-sites.js diff --git a/.gitignore b/.gitignore index 99be8a8fe..0bbd01a41 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ dist total.svg src/plugins/index.ts .history -broken-sites-report.json \ No newline at end of file +broken-sites-report.json +missed-sites-report.json \ No newline at end of file diff --git a/package.json b/package.json index 1d30a90d6..2ca759acf 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "format": "prettier --write \"./**/*.{js,ts}\"", "format:check": "prettier --check \"./**/*.{js,ts}\"", "check:sites": "node scripts/check-plugin-sites.js", + "mark:sites": "node scripts/mark-plugin-sites.js", "prepare": "husky" }, "author": "LNReader", diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js new file mode 100644 index 000000000..c52302015 --- /dev/null +++ b/scripts/mark-plugin-sites.js @@ -0,0 +1,214 @@ +#!/usr/bin/env node + +import { fileURLToPath } from 'url'; +import fs from 'fs'; +import path, { dirname } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const reportFile = path.join(__dirname, '..', 'broken-sites-report.json'); +const missedFile = path.join(__dirname, '..', 'missed-sites-report.json'); +const pluginDir = path.join(__dirname, '..', 'plugins'); + +// Language lookup likely incomplete +const langLookup = { + "‎العربية": "arabic", + "chinese": "chinese", + "English": "english", + "Français": "french", + "Bahasa Indonesia": "indonesian", + "japanese": "japanese", + "korean": "korean", + "polish": "polish", + "Português": "portuguese", + "Русский": "russian", + "Español": "spanish", + "thai": "thai", + "Türkçe": "turkish", + "Українська": "ukrainian", + "Tiếng Việt": "vietnamese", +} + +async function renameFile(oldFile, newFileInject = '.broken') { + try { + const fileToRename = path.basename(oldFile); + const directory = path.dirname(oldFile); + const ext = path.extname(fileToRename); + const fileBaseName = path.basename(oldFile, ext); + + if (fileBaseName.toString().includes(newFileInject)) { + return true + } + + // Inject newFileInject into file name + const finalNewName = fileBaseName + newFileInject + ext; + + // Construct full path + const newPath = path.join(directory, finalNewName); + + // Rename the file + await fs.renameSync(oldFile, newPath); + + console.log(`Successfully renamed: ${fileToRename} -> ${finalNewName}`); + return true; + + } catch (error) { + console.error('Error:', error.message); + return false; + } +} + +async function searchFilesForString(directory, searchString, options = {}) { + const { + recursive = false, + caseSensitive = false, + fileExtensions = ['.ts', '.js'], + showLineNumbers = false + } = options; + + const results = []; + + async function searchDirectory(dir) { + try { + const entries = await fs.readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory() && recursive) { + await searchDirectory(fullPath); + } else if (entry.isFile()) { + // Check file extension filter + if (fileExtensions && !fileExtensions.includes(path.extname(entry.name))) { + continue; + } + + try { + const content = await fs.readFileSync(fullPath, 'utf8'); + const searchTerm = caseSensitive ? searchString : searchString.toLowerCase(); + const searchContent = caseSensitive ? content : content.toLowerCase(); + + if (searchContent.includes(searchTerm)) { + const fileResult = { + file: fullPath, + matches: [] + }; + + if (showLineNumbers) { + const lines = content.split('\n'); + lines.forEach((line, index) => { + const checkLine = caseSensitive ? line : line.toLowerCase(); + if (checkLine.includes(searchTerm)) { + fileResult.matches.push({ + line: index + 1, + content: line.trim() + }); + } + }); + } + + results.push(fileResult); + } + } catch (err) { + // Skip files that can't be read as text + if (err.code !== 'EISDIR') { + console.warn(`Could not read file: ${fullPath}`); + } + } + } + } + } catch (err) { + console.error(`Error reading directory ${dir}:`, err.message); + } + } + + await searchDirectory(directory); + return results; +} + +function displayResults(results, searchString) { + if (results.length === 0) { + console.log(`\nNo files found containing "${searchString}"\n`); + return; + } + + console.log(`\nFound "${searchString}" in ${results.length} file(s):\n`); + + results.forEach(result => { + console.log(`📄 ${result.file}`); + + if (result.matches.length > 0) { + result.matches.forEach(match => { + console.log(` Line ${match.line}: ${match.content}`); + }); + } + console.log(''); + }); +} + + +async function main() { + console.log(`Loading ${reportFile}...\n`); + + try { + // Read and parse JSON file + const jsonData = await fs.readFileSync(reportFile, 'utf8', (err) => err && console.error(err)); + const data = JSON.parse(jsonData); + const broken = data.brokenSites; + + console.log(`Successfully indexed ${broken.length} plugins.\n`); + + const missed = []; + + for (const site of broken) { + const lang = langLookup[site.lang]; + const url = site.url; + + const langDir = path.join(pluginDir, lang); + + const results = await searchFilesForString(langDir, url); + + if (results.length == 0) { + console.warn(`No match for ${url}`); + missed.push(site); + continue; + } + for (const result of results) { + if (!(await renameFile(result.file))) { + missed.push(site); + } + } + } + + if (missed.length > 0) { + console.log('\n' + '='.repeat(80)); + console.log('Missed:'); + console.log('='.repeat(80)); + + fs.writeFileSync( + missedFile, + JSON.stringify( + { + timestamp: new Date().toISOString(), + total: missed.length, + brokenSites: missed, + }, + null, + 2, + ), + ); + + console.log(`\n\nDetailed report saved to: ${missedFile}`); + } else { + console.log('\n✓ All sites marked!'); + } + } catch (error) { + console.error('Error:', error.message); + process.exit(1); + } +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); From eec6144ba18b8a63740e2960da1725292b424ff5 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Sun, 11 Jan 2026 20:51:12 -0500 Subject: [PATCH 2/7] feat: Mark multisrc --- scripts/mark-plugin-sites.js | 138 ++++++++++++++++++++++++++++------- 1 file changed, 110 insertions(+), 28 deletions(-) diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js index c52302015..c5947ab16 100644 --- a/scripts/mark-plugin-sites.js +++ b/scripts/mark-plugin-sites.js @@ -12,22 +12,22 @@ const pluginDir = path.join(__dirname, '..', 'plugins'); // Language lookup likely incomplete const langLookup = { - "‎العربية": "arabic", - "chinese": "chinese", - "English": "english", - "Français": "french", - "Bahasa Indonesia": "indonesian", - "japanese": "japanese", - "korean": "korean", - "polish": "polish", - "Português": "portuguese", - "Русский": "russian", - "Español": "spanish", - "thai": "thai", - "Türkçe": "turkish", - "Українська": "ukrainian", - "Tiếng Việt": "vietnamese", -} + '‎العربية': 'arabic', + 'chinese': 'chinese', + 'English': 'english', + 'Français': 'french', + 'Bahasa Indonesia': 'indonesian', + 'japanese': 'japanese', + 'korean': 'korean', + 'polish': 'polish', + 'Português': 'portuguese', + 'Русский': 'russian', + 'Español': 'spanish', + 'thai': 'thai', + 'Türkçe': 'turkish', + 'Українська': 'ukrainian', + 'Tiếng Việt': 'vietnamese', +}; async function renameFile(oldFile, newFileInject = '.broken') { try { @@ -36,8 +36,12 @@ async function renameFile(oldFile, newFileInject = '.broken') { const ext = path.extname(fileToRename); const fileBaseName = path.basename(oldFile, ext); + // Ignore generated files if they exist + if (fileToRename.toString().includes('].ts')) { + return false; + } if (fileBaseName.toString().includes(newFileInject)) { - return true + return true; } // Inject newFileInject into file name @@ -51,7 +55,42 @@ async function renameFile(oldFile, newFileInject = '.broken') { console.log(`Successfully renamed: ${fileToRename} -> ${finalNewName}`); return true; + } catch (error) { + console.error('Error:', error.message); + return false; + } +} +async function updateMultisrc(srcFile, url, optionKey = 'broken') { + try { + // Read and parse JSON file + const jsonData = await fs.readFileSync( + srcFile, + 'utf8', + err => err && console.error(err), + ); + const data = JSON.parse(jsonData); + const date = new Date(); + const dateWithoutTime = date.toISOString().split('T')[0]; + + for (const source of data) { + const siteUrl = source.sourceSite; + if (siteUrl != url) { + continue; + } + if (!source.hasOwnProperty('options')) { + source.options = {}; + } + source.options.down = true; + source.options.downSince = dateWithoutTime; + break; + } + + // Write data back to file with previous formatting and a return + await fs.writeFileSync(srcFile, JSON.stringify(data, null, 2) + '\n'); + + console.log(`Successfully rewrote: ${srcFile} for ${url}`); + return true; } catch (error) { console.error('Error:', error.message); return false; @@ -61,9 +100,10 @@ async function renameFile(oldFile, newFileInject = '.broken') { async function searchFilesForString(directory, searchString, options = {}) { const { recursive = false, + multisrc = false, caseSensitive = false, - fileExtensions = ['.ts', '.js'], - showLineNumbers = false + fileExtensions = ['.ts', '.js', '.json'], + showLineNumbers = false, } = options; const results = []; @@ -79,19 +119,32 @@ async function searchFilesForString(directory, searchString, options = {}) { await searchDirectory(fullPath); } else if (entry.isFile()) { // Check file extension filter - if (fileExtensions && !fileExtensions.includes(path.extname(entry.name))) { + if ( + fileExtensions && + !fileExtensions.includes(path.extname(entry.name)) + ) { + continue; + } else if ( + entry.isFile() && + multisrc && + path.basename(entry.name) != 'sources.json' + ) { continue; } try { const content = await fs.readFileSync(fullPath, 'utf8'); - const searchTerm = caseSensitive ? searchString : searchString.toLowerCase(); - const searchContent = caseSensitive ? content : content.toLowerCase(); + const searchTerm = caseSensitive + ? searchString + : searchString.toLowerCase(); + const searchContent = caseSensitive + ? content + : content.toLowerCase(); if (searchContent.includes(searchTerm)) { const fileResult = { file: fullPath, - matches: [] + matches: [], }; if (showLineNumbers) { @@ -101,7 +154,7 @@ async function searchFilesForString(directory, searchString, options = {}) { if (checkLine.includes(searchTerm)) { fileResult.matches.push({ line: index + 1, - content: line.trim() + content: line.trim(), }); } }); @@ -146,19 +199,23 @@ function displayResults(results, searchString) { }); } - async function main() { console.log(`Loading ${reportFile}...\n`); try { // Read and parse JSON file - const jsonData = await fs.readFileSync(reportFile, 'utf8', (err) => err && console.error(err)); + const jsonData = await fs.readFileSync( + reportFile, + 'utf8', + err => err && console.error(err), + ); const data = JSON.parse(jsonData); const broken = data.brokenSites; console.log(`Successfully indexed ${broken.length} plugins.\n`); const missed = []; + const retry = []; for (const site of broken) { const lang = langLookup[site.lang]; @@ -170,12 +227,37 @@ async function main() { if (results.length == 0) { console.warn(`No match for ${url}`); - missed.push(site); + retry.push(site); continue; } for (const result of results) { - if (!(await renameFile(result.file))) { + const success = await renameFile(result.file); + if (success === false) { + console.warn(`Bad rename for ${url}`); + retry.push(site); + } + } + } + + if (retry.length > 0) { + for (const site of retry) { + const lang = 'multisrc'; + const url = site.url; + + const langDir = path.join(pluginDir, lang); + const options = { recursive: true, multisrc: true }; + + const results = await searchFilesForString(langDir, url, options); + + if (results.length == 0) { + console.warn(`No match for ${url}`); missed.push(site); + continue; + } + for (const result of results) { + if (!(await updateMultisrc(result.file, url))) { + missed.push(site); + } } } } From 553dd6092de64105b4816b00535f266b5c093fca Mon Sep 17 00:00:00 2001 From: Rojikku Date: Sun, 11 Jan 2026 21:31:27 -0500 Subject: [PATCH 3/7] fix: Use languages.js --- scripts/mark-plugin-sites.js | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js index c5947ab16..bd74bc5fe 100644 --- a/scripts/mark-plugin-sites.js +++ b/scripts/mark-plugin-sites.js @@ -3,6 +3,7 @@ import { fileURLToPath } from 'url'; import fs from 'fs'; import path, { dirname } from 'path'; +import languages from './languages.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -10,24 +11,16 @@ const reportFile = path.join(__dirname, '..', 'broken-sites-report.json'); const missedFile = path.join(__dirname, '..', 'missed-sites-report.json'); const pluginDir = path.join(__dirname, '..', 'plugins'); -// Language lookup likely incomplete -const langLookup = { - '‎العربية': 'arabic', - 'chinese': 'chinese', - 'English': 'english', - 'Français': 'french', - 'Bahasa Indonesia': 'indonesian', - 'japanese': 'japanese', - 'korean': 'korean', - 'polish': 'polish', - 'Português': 'portuguese', - 'Русский': 'russian', - 'Español': 'spanish', - 'thai': 'thai', - 'Türkçe': 'turkish', - 'Українська': 'ukrainian', - 'Tiếng Việt': 'vietnamese', -}; +// Languages.js is backwards from what I want +function swap(json) { + var ret = {}; + for (var key in json) { + ret[json[key]] = key; + } + return ret; +} + +const langLookup = swap(languages); async function renameFile(oldFile, newFileInject = '.broken') { try { From fe04575aade34ba11b734b6610ee7447d641439d Mon Sep 17 00:00:00 2001 From: Rojikku Date: Sun, 11 Jan 2026 21:37:36 -0500 Subject: [PATCH 4/7] fix: Correct case for langs --- scripts/mark-plugin-sites.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js index bd74bc5fe..16907ceff 100644 --- a/scripts/mark-plugin-sites.js +++ b/scripts/mark-plugin-sites.js @@ -15,7 +15,7 @@ const pluginDir = path.join(__dirname, '..', 'plugins'); function swap(json) { var ret = {}; for (var key in json) { - ret[json[key]] = key; + ret[json[key]] = key.toLowerCase(); } return ret; } From 13aaad05edec2966ed480a84ccfa9a26f64c3811 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Sun, 11 Jan 2026 21:41:59 -0500 Subject: [PATCH 5/7] chore: Remove unused option, add comments --- scripts/mark-plugin-sites.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js index 16907ceff..48f0e355f 100644 --- a/scripts/mark-plugin-sites.js +++ b/scripts/mark-plugin-sites.js @@ -54,7 +54,7 @@ async function renameFile(oldFile, newFileInject = '.broken') { } } -async function updateMultisrc(srcFile, url, optionKey = 'broken') { +async function updateMultisrc(srcFile, url) { try { // Read and parse JSON file const jsonData = await fs.readFileSync( @@ -71,6 +71,8 @@ async function updateMultisrc(srcFile, url, optionKey = 'broken') { if (siteUrl != url) { continue; } + // Set JSON modifications in here + // Checking for existence of options if (!source.hasOwnProperty('options')) { source.options = {}; } From bb454fc0e38ccdd0023ec285236ae4c157a4a423 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Tue, 13 Jan 2026 02:30:07 -0500 Subject: [PATCH 6/7] fix: Change to epoch dates --- scripts/mark-plugin-sites.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js index 48f0e355f..643bb7463 100644 --- a/scripts/mark-plugin-sites.js +++ b/scripts/mark-plugin-sites.js @@ -63,8 +63,7 @@ async function updateMultisrc(srcFile, url) { err => err && console.error(err), ); const data = JSON.parse(jsonData); - const date = new Date(); - const dateWithoutTime = date.toISOString().split('T')[0]; + const date = Date.now(); for (const source of data) { const siteUrl = source.sourceSite; @@ -77,7 +76,7 @@ async function updateMultisrc(srcFile, url) { source.options = {}; } source.options.down = true; - source.options.downSince = dateWithoutTime; + source.options.downSince = date; break; } From ee42f85cac420cf266eb7f9615610c3210d6afc9 Mon Sep 17 00:00:00 2001 From: Rojikku Date: Tue, 13 Jan 2026 18:26:46 -0500 Subject: [PATCH 7/7] fix: Don't change down time if site still down --- scripts/mark-plugin-sites.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/mark-plugin-sites.js b/scripts/mark-plugin-sites.js index 643bb7463..e1cf2ac5a 100644 --- a/scripts/mark-plugin-sites.js +++ b/scripts/mark-plugin-sites.js @@ -75,6 +75,12 @@ async function updateMultisrc(srcFile, url) { if (!source.hasOwnProperty('options')) { source.options = {}; } + if ( + source.options.hasOwnProperty('down') && + source.options.down === true + ) { + return true; + } source.options.down = true; source.options.downSince = date; break;