fix: add async-mutex to builds on large vaults

This commit is contained in:
Jacky Zhao 2023-08-20 14:27:44 -07:00
parent b99d4cd8ce
commit e65ea48fae
5 changed files with 29 additions and 6 deletions

14
package-lock.json generated
View File

@ -12,6 +12,7 @@
"@clack/prompts": "^0.6.3", "@clack/prompts": "^0.6.3",
"@floating-ui/dom": "^1.4.0", "@floating-ui/dom": "^1.4.0",
"@napi-rs/simple-git": "^0.1.8", "@napi-rs/simple-git": "^0.1.8",
"async-mutex": "^0.4.0",
"chalk": "^4.1.2", "chalk": "^4.1.2",
"chokidar": "^3.5.3", "chokidar": "^3.5.3",
"cli-spinner": "^0.2.10", "cli-spinner": "^0.2.10",
@ -1628,6 +1629,14 @@
"url": "https://github.com/sponsors/wooorm" "url": "https://github.com/sponsors/wooorm"
} }
}, },
"node_modules/async-mutex": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.4.0.tgz",
"integrity": "sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==",
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/asynckit": { "node_modules/asynckit": {
"version": "0.4.0", "version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -5582,6 +5591,11 @@
"url": "https://github.com/sponsors/wooorm" "url": "https://github.com/sponsors/wooorm"
} }
}, },
"node_modules/tslib": {
"version": "2.6.2",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
"integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
},
"node_modules/tsx": { "node_modules/tsx": {
"version": "3.12.7", "version": "3.12.7",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-3.12.7.tgz", "resolved": "https://registry.npmjs.org/tsx/-/tsx-3.12.7.tgz",

View File

@ -35,6 +35,7 @@
"@clack/prompts": "^0.6.3", "@clack/prompts": "^0.6.3",
"@floating-ui/dom": "^1.4.0", "@floating-ui/dom": "^1.4.0",
"@napi-rs/simple-git": "^0.1.8", "@napi-rs/simple-git": "^0.1.8",
"async-mutex": "^0.4.0",
"chalk": "^4.1.2", "chalk": "^4.1.2",
"chokidar": "^3.5.3", "chokidar": "^3.5.3",
"cli-spinner": "^0.2.10", "cli-spinner": "^0.2.10",

View File

@ -16,6 +16,7 @@ import http from "http"
import serveHandler from "serve-handler" import serveHandler from "serve-handler"
import { WebSocketServer } from "ws" import { WebSocketServer } from "ws"
import { randomUUID } from "crypto" import { randomUUID } from "crypto"
import { Mutex } from "async-mutex"
const ORIGIN_NAME = "origin" const ORIGIN_NAME = "origin"
const UPSTREAM_NAME = "upstream" const UPSTREAM_NAME = "upstream"
@ -391,8 +392,10 @@ See the [documentation](https://quartz.jzhao.xyz) for how to get started.
], ],
}) })
const buildMutex = new Mutex()
const timeoutIds = new Set() const timeoutIds = new Set()
const build = async (clientRefresh) => { const build = async (clientRefresh) => {
await buildMutex.acquire()
const result = await ctx.rebuild().catch((err) => { const result = await ctx.rebuild().catch((err) => {
console.error(`${chalk.red("Couldn't parse Quartz configuration:")} ${fp}`) console.error(`${chalk.red("Couldn't parse Quartz configuration:")} ${fp}`)
console.log(`Reason: ${chalk.grey(err)}`) console.log(`Reason: ${chalk.grey(err)}`)
@ -415,6 +418,7 @@ See the [documentation](https://quartz.jzhao.xyz) for how to get started.
const { default: buildQuartz } = await import(cacheFile + `?update=${randomUUID()}`) const { default: buildQuartz } = await import(cacheFile + `?update=${randomUUID()}`)
await buildQuartz(argv, clientRefresh) await buildQuartz(argv, clientRefresh)
clientRefresh() clientRefresh()
buildMutex.release()
} }
const rebuild = (clientRefresh) => { const rebuild = (clientRefresh) => {

View File

@ -16,6 +16,7 @@ import { Argv, BuildCtx } from "./util/ctx"
import { glob, toPosixPath } from "./util/glob" import { glob, toPosixPath } from "./util/glob"
import { trace } from "./util/trace" import { trace } from "./util/trace"
import { options } from "./util/sourcemap" import { options } from "./util/sourcemap"
import { Mutex } from "async-mutex"
async function buildQuartz(argv: Argv, clientRefresh: () => void) { async function buildQuartz(argv: Argv, clientRefresh: () => void) {
const ctx: BuildCtx = { const ctx: BuildCtx = {
@ -77,10 +78,11 @@ async function startServing(
} }
const initialSlugs = ctx.allSlugs const initialSlugs = ctx.allSlugs
let timeoutIds: Set<ReturnType<typeof setTimeout>> = new Set() const buildMutex = new Mutex()
let toRebuild: Set<FilePath> = new Set() const timeoutIds: Set<ReturnType<typeof setTimeout>> = new Set()
let toRemove: Set<FilePath> = new Set() const toRebuild: Set<FilePath> = new Set()
let trackedAssets: Set<FilePath> = new Set() const toRemove: Set<FilePath> = new Set()
const trackedAssets: Set<FilePath> = new Set()
async function rebuild(fp: string, action: "add" | "change" | "delete") { async function rebuild(fp: string, action: "add" | "change" | "delete") {
// don't do anything for gitignored files // don't do anything for gitignored files
if (ignored(fp)) { if (ignored(fp)) {
@ -111,6 +113,7 @@ async function startServing(
// debounce rebuilds every 250ms // debounce rebuilds every 250ms
timeoutIds.add( timeoutIds.add(
setTimeout(async () => { setTimeout(async () => {
await buildMutex.acquire()
const perf = new PerfTimer() const perf = new PerfTimer()
console.log(chalk.yellow("Detected change, rebuilding...")) console.log(chalk.yellow("Detected change, rebuilding..."))
try { try {
@ -143,6 +146,7 @@ async function startServing(
clientRefresh() clientRefresh()
toRebuild.clear() toRebuild.clear()
toRemove.clear() toRemove.clear()
buildMutex.release()
}, 250), }, 250),
) )
} }

View File

@ -25,13 +25,13 @@ export default ((userOpts?: Partial<Options>) => {
const opts = { ...defaultOptions, ...userOpts } const opts = { ...defaultOptions, ...userOpts }
function RecentNotes(props: QuartzComponentProps) { function RecentNotes(props: QuartzComponentProps) {
const { allFiles, fileData, displayClass } = props const { allFiles, fileData, displayClass } = props
const pages = allFiles.filter(opts.filter).sort(opts.sort).slice(0, opts.limit) const pages = allFiles.filter(opts.filter).sort(opts.sort)
const remaining = Math.max(0, pages.length - opts.limit) const remaining = Math.max(0, pages.length - opts.limit)
return ( return (
<div class={`recent-notes ${displayClass}`}> <div class={`recent-notes ${displayClass}`}>
<h3>{opts.title}</h3> <h3>{opts.title}</h3>
<ul class="recent-ul"> <ul class="recent-ul">
{pages.map((page) => { {pages.slice(0, opts.limit).map((page) => {
const title = page.frontmatter?.title const title = page.frontmatter?.title
const tags = page.frontmatter?.tags ?? [] const tags = page.frontmatter?.tags ?? []