main.lua

-- main.lua — gitloom: a git hosting service on the xnet2lua runtime.
--
-- Current scope: the git smart-HTTP transport (clone / fetch / push), a
-- repository model on disk, HTTP Basic accounts with access tokens, a JSON
-- management API, a basic issue tracker, and the first same-origin repository
-- browser. Pull requests come later and ride on the same API.
--
-- Run:  bin\xnet.exe main.lua [KEY=VAL ...]      (see gitloom.cfg for keys)
--
-- ARCHITECTURE
--   main thread            the event loop: accept, HTTP parse, routing, and one
--                          coroutine per request
--   WORKER_GRP1..+N        the xproc pool, where every `git` invocation blocks
--
-- Nothing shares state across that boundary: a request coroutine yields on an
-- RPC into a worker, the worker runs one command to completion, and the reply
-- comes back through xrouter. That is why every handler that touches git must
-- run on a coroutine, and why nothing in this file may call git during load.
--
-- MODULE CONVENTION
-- app/*.lua run in separate environments, and that isolation comes from the
-- loader, not from `dofile`: bare top-level names stay private to their file
-- only when it is loaded through boot.load_script. Public functions are
-- installed on the one gitloom global, g_exports, and every module reaches an
-- earlier module's API by its short name.
--
-- THIS FILE RUNS TWICE
-- The runtime loads main.lua into the root `_G`, where the short names below —
-- cfg_log_error, proc_selftest, repo_root — do not resolve. So the first pass
-- does nothing except load the loader and hand main.lua straight back to it.
-- The second pass arrives with `boot` as a chunk argument, already inside an
-- app environment, and falls through the guard into the real work.
--
-- The point of the detour is that neither `setfenv` (5.1) nor a lexical `_ENV`
-- (5.2+) appears here. app/boot.lua is the only file that knows the difference,
-- which matters because a build can only ever execute one of the two branches:
-- bin/xnet.exe is minilua (5.5), and LUA_BACKEND=luajit is 5.1.

local boot = ...
if type(boot) ~= 'table' then     -- pass 1: the runtime passes no arguments
    boot = dofile('app/boot.lua')
    return boot.run_script('main.lua', boot)
end

boot.load_script('app/cfg.lua')      -- cfg_*                       (no deps)
boot.load_script('app/util.lua')     -- util_*
boot.load_script('app/db.lua')       -- db_*                        (mysql)
boot.load_script('app/migrate.lua')  -- migrate_*                   (schema)
boot.load_script('app/store.lua')    -- store_*   accounts and repositories
boot.load_script('app/proc.lua')     -- proc_*
boot.load_script('app/pkt.lua')      -- pkt_*
boot.load_script('app/repo.lua')     -- repo_*
boot.load_script('app/issue.lua')    -- issue_*
boot.load_script('app/org.lua')      -- organization membership and team grants
boot.load_script('app/pr.lua')       -- pull requests and reviews
boot.load_script('app/protect.lua')  -- protect_*   what a push may not do
boot.load_script('app/git.lua')      -- git_*
-- Before auth.lua, and not only by convention: namespace ownership is decided
-- against the modules already loaded, so auth.lua could otherwise claim
-- auth_ratelimit_* and nothing would stop it.
boot.load_script('app/auth_ratelimit.lua')  -- auth_ratelimit_*
boot.load_script('app/auth.lua')     -- auth_*
boot.load_script('app/http.lua')     -- http_*
boot.load_script('app/stream.lua')   -- stream_*, stream_response
boot.load_script('app/browse.lua')   -- browse_*
boot.load_script('app/smart.lua')    -- smart_install
boot.load_script('app/api.lua')      -- api_install
boot.load_script('app/web.lua')      -- web_install

-- xrouter carries the xproc workers' RPC replies back to this thread. Exposing
-- its handler as __thread_handle is the only wiring that needs; without it
-- every proc_exec() would sit until its RPC deadline and report a transport
-- failure with no explanation.
local router = dofile('scripts/core/share/xrouter.lua')
router.set_log_prefix('GITLOOM')

local sweep_timer = nil
local listen_timer = nil
local listen_now = false

-- ---------------------------------------------------------------------------
-- Boot work that has to run on a coroutine
--
-- Anything that shells out yields, and __init runs on the main state where a
-- yield is not possible. So the checks that need git go here and are kicked off
-- as a coroutine at the end of __init. A failure is fatal on purpose: a gitloom
-- that cannot run git will answer every clone with a 500, and finding out at
-- boot beats finding out from a user.
-- ---------------------------------------------------------------------------
local function boot_async()
    -- The database first: everything else that boots is cheaper to redo than a
    -- half-migrated schema, and a gitloom pointed at a database it cannot reach
    -- has nothing to serve. Here rather than in __init because every statement
    -- yields on an RPC into the mysql worker thread.
    if db_enabled() then
        local applied, merr = migrate_run()
        if not applied then
            cfg_log_error('schema migration failed: %s', tostring(merr))
            xthread.stop(1)
            return
        end
        local st = migrate_status()
        cfg_log_system('schema at migration %d (%d applied this boot)',
            st and st.latest or migrate_latest(), applied)
    end

    local ok, err = proc_selftest()
    if not ok then
        cfg_log_error('process pool self-test failed: %s', tostring(err))
        xthread.stop(1)
        return
    end

    local version, verr = git_version()
    if not version then
        cfg_log_error('git is not usable (GIT_BIN=%q): %s', git_bin(), tostring(verr))
        cfg_log_error('gitloom drives the real git binary; install git or set GIT_BIN')
        xthread.stop(1)
        return
    end
    local vok, want = git_version_ok(version)
    if not vok then
        cfg_log_error('git %s is too old; gitloom needs >= %s', version, want)
        xthread.stop(1)
        return
    end
    -- Cache it: /api/v1/version is reachable without credentials, and
    -- re-running git per request let anyone occupy the whole process pool
    -- from outside.
    git_version_cache_set(version)
    cfg_log_system('git %s via %q', version, git_bin())
    git_stream_report()

    -- Branch protection, and it refuses to boot rather than warning. The thing
    -- being guarded against is an instance that reports itself healthy while
    -- every protected branch is open to a force-push; an operator who wants to
    -- run without it says so with PROTECT_DEFAULT_BRANCH=off, and that is a
    -- decision rather than an accident. After git, because it is git that has
    -- to run the hook.
    if cfg_bool('PROTECT_DEFAULT_BRANCH', true) then
        local pok, perr = protect_setup()
        if not pok then
            cfg_log_error('branch protection is on and cannot be armed: %s', tostring(perr))
            cfg_log_error('point HOOKS_DIR at the directory holding `update`, ' ..
                          'or set PROTECT_DEFAULT_BRANCH=off to run without it')
            xthread.stop(1)
            return
        end
        cfg_log_system('branch protection: default branches guarded, hooks at %q',
            protect_hooks_dir())
    else
        cfg_log_warn('branch protection is OFF — a force-push may rewrite any branch')
    end
    -- Says whether X-Forwarded-For is honoured, and from whom. Silent
    -- misconfiguration here makes every audit line and every rate-limit
    -- decision name the proxy instead of the client.
    http_trusted_proxy_report()

    -- Debris from a previous run: a crash mid-clone leaves a half-written
    -- packfile nothing will ever come back for. Safe here because no request
    -- has been served yet.
    proc_tmp_purge()

    -- The stores, and only now: under MySQL every one of these is a query, and
    -- __init runs on the main state where a yield is not possible.
    if not repo_index_load() then xthread.stop(1); return end
    if not issue_index_load() then xthread.stop(1); return end
    if not org_index_load() then xthread.stop(1); return end
    if not pr_index_load() then xthread.stop(1); return end
    if not auth_load() then xthread.stop(1); return end
    auth_bootstrap()
    cfg_log_system('store: %s', store_describe())

    -- LAST. Nothing is accepted until git has been checked and the stores have
    -- answered, so there is no window in which a clone is served by an instance
    -- that does not yet know which repositories exist.
    --
    -- Through a main-state timer rather than from here: http_listen arms the
    -- serve loop's ticker, and a timer armed inside this coroutine would fire
    -- into it after it has finished. The same rule as the scratch sweeper.
    listen_now = true

    local n = #repo_list()
    cfg_log_system('gitloom ready — %d repository(ies) under %s', n, repo_root())
end

-- ---------------------------------------------------------------------------

local function __init()
    assert(xnet.init())
    xtimer.init(16)

    util_dir_make(cfg_get('DATA_DIR', 'data'))
    util_dir_make(cfg_get('TMP_DIR', 'tmp'))
    util_dir_make(repo_root())

    local ok, err = proc_setup()
    if not ok then error('process pool failed to start: ' .. tostring(err)) end

    -- Password hashing gets its own thread: PBKDF2 is deliberately slow, and on
    -- the event loop it made bad logins a denial-of-service against clones.
    ok, err = auth_kdf_setup()
    if not ok then error('kdf worker failed to start: ' .. tostring(err)) end

    -- The mysql pool is a thread too, and threads are created HERE for the same
    -- reason timers are: xthread hands the caller a ThreadData userdata owned by
    -- the calling lua_State, and its __gc nulls that struct. Started inside the
    -- boot coroutine, the worker would be reclaimed out from under itself the
    -- moment that coroutine was collected. Connecting is asynchronous, so
    -- nothing is proven here — migrate_run in boot_async is the first statement
    -- and therefore the real check.
    if db_enabled() then
        ok, err = db_start()
        if not ok then error('mysql pool failed to start: ' .. tostring(err)) end
    end

    -- git children run with HOME pointed here so they cannot pick up the
    -- operator's ~/.gitconfig. The directory has to exist or git warns on every
    -- invocation.
    util_dir_make(git_home_dir())

    smart_install()   -- git transport, as a path fallback
    web_install()     -- same-origin repository browser, as routes
    api_install()     -- JSON API, as routes
    -- Order matters only in that both must be registered before the listener
    -- accepts anything; the route table is always consulted before fallbacks.

    -- Armed here, on the main state; it fires once boot_async has set the flag.
    -- See the note at the end of boot_async for why the listen cannot simply
    -- happen there.
    listen_timer = xtimer.add(20, function()
        if not listen_now then return end
        listen_timer:del(); listen_timer = nil
        local lok, lerr = http_listen()
        if not lok then
            cfg_log_error('listen failed: %s', tostring(lerr))
            xthread.stop(1)
        end
    end, -1)

    -- Scratch files released by finished responses are unlinked here. Pure Lua,
    -- no shelling out, so it is safe to run straight off the timer. Armed from
    -- the main state: xtimer keeps a raw pointer to whichever lua_State armed
    -- a timer, and one armed inside a request coroutine fires into freed memory
    -- once that coroutine is collected.
    sweep_timer = xtimer.add(cfg_int('TMP_SWEEP_MS', 60000), function()
        proc_tmp_sweep()
        -- Rate-limit buckets are created per source address, so on a public
        -- instance the table grows without bound unless stale ones are dropped.
        auth_ratelimit_gc()
    end, -1)

    -- Lock _G. From here a mistyped global raises instead of evaluating to nil.
    -- After every module has loaded and before any request can arrive.
    if cfg_bool('STRICT_GLOBALS', true) then
        boot.strict_enable()
        cfg_log_info('strict globals on')
    end

    -- pcall INSIDE the coroutine, not around the resume: boot_async yields, so
    -- the resume returns at the first yield and a raise after that point would
    -- reach nobody. It used to be survivable — the listener was already up — but
    -- now the listener is what boot_async switches on, so an unreported failure
    -- here is a process that accepts nothing and logs nothing about why.
    local resumed, rerr = coroutine.resume(coroutine.create(function()
        local bok, berr = pcall(boot_async)
        if not bok then
            cfg_log_error('boot failed: %s', tostring(berr))
            xthread.stop(1)
        end
    end))
    if not resumed then
        cfg_log_error('boot coroutine crashed: %s', tostring(rerr))
        xthread.stop(1)
    end
end

local function __uninit()
    if sweep_timer then sweep_timer:del(); sweep_timer = nil end
    if listen_timer then listen_timer:del(); listen_timer = nil end
    db_stop()
    http_close()
    proc_shutdown()     -- join the git worker pool while this state is still valid
    xnet.uninit()
    cfg_log_system('gitloom stopped')
end

return {
    __thread_handle = router.handle,
    __init = __init,
    __uninit = __uninit,
}