coverage — ~bigbes/sr-ht-dolt — ede3b0bb — storage/init.go

Coverage
78.6% 55/70 statements
Δ
—
Blob
b192dfd
Uncovered L71-L72L78-L79L82-L84L89-L90L103-L104L107-L108L112-L113L124-L125L127-L128L153-L154L172-L173L210-L211L217-L218L220-L221
1 // Package storage owns dolt.sr.ht's on-disk NBS chunk stores and the
2 // remotesapi DBCache that serves them.
3 //
4 // A hosted database is a bare NBS chunk-store directory (no ".dolt/", no
5 // working set) laid out at "<root>/~<owner>/<name>". This is exactly what
6 // remotesrv serves and what "file://" dolt remotes consume, so InitStore can
7 // create one with the low-level doltdb primitives and remotesrv can read and
8 // write it directly.
9 //
10 // # remotesrv FS working directory (load-bearing)
11 //
12 // remotesrv seals its chunk-download URLs relative to the working directory of
13 // the filesys it is given. The server MUST be constructed with
14 // filesys.LocalFilesysWithWorkingDir(root) pointing at the repos root — NOT a
15 // plain filesys.LocalFS. With a working-dir-rooted FS the sealed URLs carry
16 // clean relative prefixes (e.g. "~owner/db"); with a bare LocalFS they carry
17 // "../../.." escapes that the sealed-URL file handler rejects, and every clone
18 // or push breaks at the chunk-transfer stage. This was proven end-to-end in the
19 // Phase-0 spike (storage/spike_test.go). Cache in this package keys stores by
20 // absolute disk path, so it works with either FS, but the server assembly in
21 // remoteapi/ must still honor this rule.
22 package storage
23
24 import (
25 "context"
26 "fmt"
27 "os"
28 "path/filepath"
29 "strings"
30
31 "github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
32 "github.com/dolthub/dolt/go/libraries/utils/earl"
33 "github.com/dolthub/dolt/go/libraries/utils/filesys"
34 "github.com/dolthub/dolt/go/store/nbs"
35 "github.com/dolthub/dolt/go/store/types"
36 )
37
38 // RepoDiskPath returns the absolute on-disk store directory for a database,
39 // laid out as "<root>/~<owner>/<name>". Callers are responsible for validating
40 // owner and name (see core.ValidateName) before touching disk.
41 22 func RepoDiskPath(root, owner, name string) string {
42 22 return filepath.Join(root, "~"+owner, name)
43 22 }
44
45 // InitStore creates a bare NBS chunk store at absPath and writes an empty repo
46 // into it with a single "main" branch and an initial commit authored by
47 // ownerName/ownerEmail.
48 //
49 // It is the OPT-IN half of database creation — the create form's "initialize
50 // with an empty commit" checkbox — and not what any automatic path uses. The
51 // initial commit it writes is history, so a client pushing a database that has
52 // its own history is pushing a non-fast-forward and must --force. What it buys
53 // in exchange is a database that can be cloned before anything is pushed to it,
54 // which InitEmptyStore's result cannot.
55 //
56 // absPath must be absolute. On any failure after the directory is created,
57 // InitStore removes absPath so a failed creation never leaves a partial store
58 // behind. Idempotence is NOT provided: calling InitStore on an existing store
59 // is a caller error and is not defended against here.
60 12 func InitStore(ctx context.Context, absPath, ownerName, ownerEmail string) (err error) {
61 12 if !filepath.IsAbs(absPath) {
62 1 return fmt.Errorf("storage: InitStore requires an absolute path, got %q", absPath)
63 1 }
64
65 11 if err := os.MkdirAll(absPath, 0o755); err != nil {
66 1 return fmt.Errorf("storage: create store dir %q: %w", absPath, err)
67 1 }
68 // Any error past this point must not leave a half-written store behind.
69 10 defer func() {
70 10 if err != nil {
71 0 os.RemoveAll(absPath)
72 0 }
73 }()
74
75 10 fileURL := earl.FileUrlFromPath(absPath, os.PathSeparator)
76 10 ddb, err := doltdb.LoadDoltDB(ctx, types.Format_DOLT, fileURL, filesys.LocalFS)
77 10 if err != nil {
78 0 return fmt.Errorf("storage: load doltdb at %q: %w", fileURL, err)
79 0 }
80
81 10 if err = ddb.WriteEmptyRepo(ctx, "main", ownerName, ownerEmail); err != nil {
82 0 ddb.Close()
83 0 return fmt.Errorf("storage: write empty repo at %q: %w", absPath, err)
84 0 }
85
86 // Release the init handle so the server can open its own store over the
87 // same directory later.
88 10 if err = ddb.Close(); err != nil {
89 0 return fmt.Errorf("storage: close init doltdb at %q: %w", absPath, err)
90 0 }
91 10 return nil
92 }
93
94 // InitEmptyStore creates a genuinely empty bare NBS chunk store at absPath —
95 // a directory whose store root is the empty hash, with NO commits and NO
96 // working set. Unlike InitStore it deliberately does NOT call WriteEmptyRepo:
97 // an initial "Initialize data repository" commit would make the first push to
98 // this store a non-fast-forward and be rejected. An empty store lets the
99 // client's first push land as the initial history. Used by push-to-create.
100 // absPath must be absolute; on any failure the directory is removed.
101 1 func InitEmptyStore(ctx context.Context, absPath string) (err error) {
102 1 if !filepath.IsAbs(absPath) {
103 0 return fmt.Errorf("storage: InitEmptyStore requires an absolute path, got %q", absPath)
104 0 }
105
106 1 if err := os.MkdirAll(absPath, 0o755); err != nil {
107 0 return fmt.Errorf("storage: create store dir %q: %w", absPath, err)
108 0 }
109 // Any error past this point must not leave a half-written store behind.
110 1 defer func() {
111 1 if err != nil {
112 0 os.RemoveAll(absPath)
113 0 }
114 }()
115
116 // Opening a plain (non-generational) NBS store over the freshly created,
117 // empty directory both validates the directory (checkDir requires it to
118 // exist) and confirms the store is a valid empty store (a missing manifest
119 // is treated lazily as an empty store with the null root). This mirrors the
120 // construction storage.Cache.Get uses to serve pushes; a plain store closes
121 // cleanly, unlike the generational store LoadDoltDB routes through.
122 1 cs, err := nbs.NewLocalStore(ctx, types.Format_DOLT.VersionString(), absPath, defaultMemTableSize, nbs.NewUnlimitedMemQuotaProvider(), false)
123 1 if err != nil {
124 0 return fmt.Errorf("storage: open empty store at %q: %w", absPath, err)
125 0 }
126 1 if err = cs.Close(); err != nil {
127 0 return fmt.Errorf("storage: close empty store at %q: %w", absPath, err)
128 0 }
129 1 return nil
130 }
131
132 // containedPath checks that absPath is an absolute path strictly inside the
133 // absolute root and returns both cleaned. It is the guard every destructive
134 // path operation in this package runs first, so a corrupted or
135 // attacker-controlled path can never escape the configured repos root — and
136 // the root itself is never a legal target. op names the caller ("DeleteStore",
137 // "MoveStore") so a refusal says which operation was stopped.
138 33 func containedPath(op, root, absPath string) (cleanRoot, cleanPath string, err error) {
139 33 if !filepath.IsAbs(root) {
140 2 return "", "", fmt.Errorf("storage: %s requires an absolute root, got %q", op, root)
141 2 }
142 31 if !filepath.IsAbs(absPath) {
143 3 return "", "", fmt.Errorf("storage: %s requires an absolute path, got %q", op, absPath)
144 3 }
145
146 28 cleanRoot = filepath.Clean(root)
147 28 cleanPath = filepath.Clean(absPath)
148 28 if cleanPath == cleanRoot {
149 3 return "", "", fmt.Errorf("storage: %s refuses to act on the repos root %q", op, cleanRoot)
150 3 }
151 25 rel, err := filepath.Rel(cleanRoot, cleanPath)
152 25 if err != nil {
153 0 return "", "", fmt.Errorf("storage: %s rel(%q, %q): %w", op, cleanRoot, cleanPath, err)
154 0 }
155 25 if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
156 7 return "", "", fmt.Errorf("storage: %s refuses %q outside repos root %q", op, cleanPath, cleanRoot)
157 7 }
158 18 return cleanRoot, cleanPath, nil
159 }
160
161 // DeleteStore removes the store directory at absPath. It refuses to delete
162 // anything that is not strictly contained within root, guarding against a
163 // corrupted or attacker-controlled path escaping the configured repos root.
164 // Both root and absPath must be absolute.
165 7 func DeleteStore(ctx context.Context, root, absPath string) error {
166 7 _, cleanPath, err := containedPath("DeleteStore", root, absPath)
167 7 if err != nil {
168 6 return err
169 6 }
170
171 1 if err := os.RemoveAll(cleanPath); err != nil {
172 0 return fmt.Errorf("storage: remove store %q: %w", cleanPath, err)
173 0 }
174 1 return nil
175 }
176
177 // MoveStore relocates the store directory at srcPath to dstPath, the on-disk
178 // half of a rename. Both paths must be absolute and strictly inside root.
179 //
180 // It never overwrites: an existing dstPath is refused before anything is
181 // touched, because os.Rename over an empty destination directory would succeed
182 // silently and swallow it. A missing srcPath is likewise an error rather than a
183 // no-op — a rename whose store never moved would leave the metadata row
184 // pointing at nothing.
185 //
186 // The move itself is one os.Rename, so within a filesystem it is atomic: the
187 // store is either wholly at the old path or wholly at the new one, never half
188 // copied. Open handles on the old directory survive it (the inodes move, not
189 // the files), but they keep resolving new writes against the old path string,
190 // so the caller must still evict any memoized handle — see Cache.Evict.
191 15 func MoveStore(ctx context.Context, root, srcPath, dstPath string) error {
192 15 _, cleanSrc, err := containedPath("MoveStore", root, srcPath)
193 15 if err != nil {
194 4 return err
195 4 }
196 11 _, cleanDst, err := containedPath("MoveStore", root, dstPath)
197 11 if err != nil {
198 5 return err
199 5 }
200 6 if cleanSrc == cleanDst {
201 1 return fmt.Errorf("storage: MoveStore source and destination are the same path %q", cleanSrc)
202 1 }
203
204 5 if _, err := os.Stat(cleanSrc); err != nil {
205 1 return fmt.Errorf("storage: MoveStore source %q: %w", cleanSrc, err)
206 1 }
207 4 if _, err := os.Stat(cleanDst); err == nil {
208 2 return fmt.Errorf("storage: MoveStore refuses to overwrite an existing store at %q", cleanDst)
209 2 } else if !os.IsNotExist(err) {
210 0 return fmt.Errorf("storage: MoveStore destination %q: %w", cleanDst, err)
211 0 }
212
213 // The owner directory ("<root>/~<owner>") already exists for any store that
214 // is being renamed within its owner's namespace, but creating it keeps the
215 // operation correct if a future caller ever moves a store across owners.
216 2 if err := os.MkdirAll(filepath.Dir(cleanDst), 0o755); err != nil {
217 0 return fmt.Errorf("storage: create destination dir %q: %w", filepath.Dir(cleanDst), err)
218 0 }
219 2 if err := os.Rename(cleanSrc, cleanDst); err != nil {
220 0 return fmt.Errorf("storage: move store %q -> %q: %w", cleanSrc, cleanDst, err)
221 0 }
222 2 return nil
223 }