pacenotes_server/store/
generate.rs

1//! `manifest gen`: walk `<data>/<game>/packages/<directory>/**/*.zip`, hash
2//! every zip, merge its sidecar, and collect what the publisher should fix.
3//!
4//! The result depends on the zips and sidecars alone, never on the previous
5//! manifest, so deleting a sidecar key removes the value and replicas agree.
6
7/// A zip outside an access directory or with a broken sidecar is withheld as an
8/// error rather than published with silently wrong metadata; the rest publishes.
9/// A game publishes nothing without a valid `schema.toml`, and a package in a
10/// category its schema does not declare is skipped.
11pub fn generate(data_dir: &std::path::Path) -> anyhow::Result<super::Generated> {
12    generate_with(data_dir, crate::store::BUNDLED_SCHEMAS)
13}
14
15/// `generate`, comparing each game's schema with `bundled` instead of this
16/// release's own, so tests' fixture schemas need not match the repository's.
17pub fn generate_with(
18    data_dir: &std::path::Path,
19    bundled: &[(&str, &[u8])],
20) -> anyhow::Result<super::Generated> {
21    let mut generated = super::Generated {
22        manifest: pacenotes_shared::Manifest::default(),
23        warnings: Vec::new(),
24        errors: Vec::new(),
25    };
26
27    for entry in read_dir_sorted(data_dir)? {
28        let path = entry.path();
29        if path.is_file() {
30            if is_zip(&path) {
31                generated.error(format!(
32                    "top-level zip {}: a package belongs under <game>/{}/<directory>/ — \
33                     not published",
34                    path.display(),
35                    crate::store::PACKAGES_DIR
36                ));
37            }
38            continue;
39        }
40        let game = entry.file_name().to_string_lossy().into_owned();
41        if path.is_dir() && !crate::store::RESERVED_TOP_LEVEL.contains(&game.as_str()) {
42            scan_game(&path, &game, bundled, &mut generated)?;
43        }
44    }
45
46    for problem in crate::store::requires_check::requires_problems(&generated.manifest) {
47        generated.warn(problem);
48    }
49    // Advice for the publisher from recent reports, never an input to the manifest.
50    let reports_dir = data_dir.join(crate::store::REPORTS_DIR);
51    for warning in crate::reports::overlap_warnings(&reports_dir, crate::auth::now()) {
52        generated.warn(warning);
53    }
54    Ok(generated)
55}
56
57fn scan_game(
58    game_dir: &std::path::Path,
59    game: &str,
60    bundled: &[(&str, &[u8])],
61    generated: &mut super::Generated,
62) -> anyhow::Result<()> {
63    let packages_dir = crate::store::PACKAGES_DIR;
64    if !pacenotes_shared::valid_id(game) {
65        generated.warn(format!("skipping {game:?}: a game id must be [a-z0-9_-]+"));
66        return Ok(());
67    }
68    if pacenotes_shared::KNOWN_CATEGORIES.contains(&game) || game == pacenotes_shared::PUBLIC_DIR {
69        generated.warn(format!(
70            "{game:?} is at the top level, where a GAME id belongs (e.g. {:?}). \
71             The layout is <game>/{packages_dir}/<directory>/<name>.zip.",
72            pacenotes_shared::GAME_RBR
73        ));
74    }
75    if !pacenotes_shared::KNOWN_GAMES.contains(&game) {
76        generated.warn(format!(
77            "{game:?} is not a game this protocol knows ({:?}) — publishing it anyway",
78            pacenotes_shared::KNOWN_GAMES
79        ));
80    }
81
82    let packages_root = game_dir.join(packages_dir);
83    if !packages_root.is_dir() {
84        generated.warn(format!(
85            "{game}/ has no {packages_dir}/ folder — nothing published for it"
86        ));
87        return Ok(());
88    }
89    let schema = match crate::store::game_schema::load_schema(game_dir, game) {
90        Ok(schema) => schema,
91        Err(error) => {
92            generated.error(error);
93            return Ok(());
94        }
95    };
96    if let Some(warning) = crate::store::game_schema::bundled_schema_warning(game, &schema, bundled)
97    {
98        generated.warn(warning);
99    }
100
101    for entry in read_dir_sorted(&packages_root)? {
102        let path = entry.path();
103        if path.is_file() {
104            if is_zip(&path) {
105                generated.error(format!(
106                    "zip outside an access directory: {} — every package must live in one \
107                     (put anonymous packages in {}/); not published",
108                    path.display(),
109                    pacenotes_shared::PUBLIC_DIR
110                ));
111            }
112            continue;
113        }
114        let directory = entry.file_name().to_string_lossy().into_owned();
115        if !path.is_dir() {
116            continue;
117        }
118        if !pacenotes_shared::valid_id(&directory) {
119            generated.warn(format!(
120                "skipping {game}/{packages_dir}/{directory:?}: an access directory must be \
121                 [a-z0-9_-]+"
122            ));
123            continue;
124        }
125        if pacenotes_shared::KNOWN_CATEGORIES.contains(&directory.as_str()) {
126            generated.warn(format!(
127                "{game}/{packages_dir}/{directory}/ is a CATEGORY name where an access \
128                 directory (e.g. {public:?}) belongs — categories go one level down, as \
129                 {public}/{directory}/…",
130                public = pacenotes_shared::PUBLIC_DIR
131            ));
132        }
133        scan_access_directory(&path, game, &directory, &schema, generated)?;
134    }
135    generated.manifest.schemas.insert(game.to_string(), schema);
136    Ok(())
137}
138
139fn scan_access_directory(
140    directory_path: &std::path::Path,
141    game: &str,
142    directory: &str,
143    schema: &pacenotes_shared::GameSchema,
144    generated: &mut super::Generated,
145) -> anyhow::Result<()> {
146    let mut found = FoundFiles::default();
147    collect(directory_path, "", &mut found, generated);
148
149    for (name, zip_path) in &found.zips {
150        let category = name.split('/').next().unwrap_or(name);
151        if !schema
152            .categories
153            .iter()
154            .any(|declared| declared.id == category)
155        {
156            generated.warn(format!(
157                "{game}/{directory}/{name}.zip is in category {category:?}, which \
158                 {game}/{} does not declare — not published",
159                pacenotes_shared::SCHEMA_FILE
160            ));
161            continue;
162        }
163        let sidecar = match crate::store::sidecar::read_sidecar(&zip_path.with_extension("toml")) {
164            Ok(sidecar) => sidecar,
165            Err(e) => {
166                generated.error(format!(
167                    "{game}/{directory}/{name}.toml: {e} — package not published"
168                ));
169                continue;
170            }
171        };
172        let package = package_entry(game, directory, name, zip_path, sidecar)?;
173        generated.manifest.packages.push(package);
174    }
175
176    for sidecar in &found.sidecars {
177        if !found.zips.iter().any(|(name, _)| name == sidecar) {
178            generated.warn(format!(
179                "{game}/{directory}/{sidecar}.toml has no {sidecar}.zip next to it — \
180                 the metadata is ignored"
181            ));
182        }
183    }
184    Ok(())
185}
186
187/// Metadata from the sidecar alone, with a version derived from the bytes when
188/// it names none; size, hash and mtime always come from the zip.
189fn package_entry(
190    game: &str,
191    directory: &str,
192    name: &str,
193    zip_path: &std::path::Path,
194    sidecar: crate::store::sidecar::Sidecar,
195) -> anyhow::Result<pacenotes_shared::PackageInfo> {
196    let metadata = std::fs::metadata(zip_path)
197        .map_err(|e| anyhow::anyhow!("stat {}: {e}", zip_path.display()))?;
198    let updated_at = metadata
199        .modified()
200        .ok()
201        .and_then(|modified| modified.duration_since(std::time::UNIX_EPOCH).ok())
202        .map(|since_epoch| since_epoch.as_secs() as i64)
203        .unwrap_or(0);
204
205    let mut requires = sidecar.requires.unwrap_or_default();
206    requires.sort();
207    requires.dedup();
208
209    let sha256 = crate::store::sha256_file(zip_path)?;
210    Ok(pacenotes_shared::PackageInfo {
211        directory: directory.to_string(),
212        name: name.to_string(),
213        version: sidecar
214            .version
215            .unwrap_or_else(|| crate::store::derived_version(&sha256)),
216        size: metadata.len(),
217        sha256,
218        updated_at,
219        game: game.to_string(),
220        display_name: sidecar.display_name,
221        description: sidecar.description,
222        exclusive_group: sidecar.exclusive_group,
223        requires,
224    })
225}
226
227/// The zips and sidecars under one access directory, keyed by package `name`.
228#[derive(Default)]
229struct FoundFiles {
230    zips: Vec<(String, std::path::PathBuf)>,
231    sidecars: Vec<String>,
232}
233
234/// Depth-first in name order, which keeps a regenerated manifest byte-stable.
235fn collect(
236    dir: &std::path::Path,
237    prefix: &str,
238    found: &mut FoundFiles,
239    generated: &mut super::Generated,
240) {
241    let entries = match read_dir_sorted(dir) {
242        Ok(entries) => entries,
243        Err(e) => {
244            generated.warn(format!("skipping {}: {e}", dir.display()));
245            return;
246        }
247    };
248
249    for entry in entries {
250        let path = entry.path();
251        let Some(file_name) = entry.file_name().to_str().map(str::to_string) else {
252            generated.warn(format!(
253                "skipping {}: name is not valid UTF-8",
254                path.display()
255            ));
256            continue;
257        };
258        if path.is_dir() {
259            collect(&path, &join_name(prefix, &file_name), found, generated);
260            continue;
261        }
262
263        let extension = path.extension().and_then(|extension| extension.to_str());
264        if !path.is_file() || !matches!(extension, Some("zip" | "toml")) {
265            continue;
266        }
267        let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else {
268            continue;
269        };
270        let name = join_name(prefix, stem);
271        if !pacenotes_shared::valid_name(&name) {
272            generated.warn(format!(
273                "skipping {}: {name:?} is not a usable package name",
274                path.display()
275            ));
276            continue;
277        }
278
279        if extension == Some("zip") {
280            if !url_clean(&name) {
281                generated.warn(format!(
282                    "{name:?} contains characters a client must percent-encode \
283                     (space, #, ?, %, +, …) — consider renaming the file"
284                ));
285            }
286            found.zips.push((name, path));
287        } else {
288            found.sidecars.push(name);
289        }
290    }
291}
292
293/// Does every byte pass through a URL unescaped, `/` included?
294fn url_clean(name: &str) -> bool {
295    name.bytes()
296        .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~' | b'/'))
297}
298
299/// Joins with `/` on every host: a name is a protocol string, not a filesystem path.
300fn join_name(prefix: &str, segment: &str) -> String {
301    if prefix.is_empty() {
302        segment.to_string()
303    } else {
304        format!("{prefix}/{segment}")
305    }
306}
307
308fn is_zip(path: &std::path::Path) -> bool {
309    path.extension().is_some_and(|extension| extension == "zip")
310}
311
312fn read_dir_sorted(dir: &std::path::Path) -> anyhow::Result<Vec<std::fs::DirEntry>> {
313    let mut entries = std::fs::read_dir(dir)
314        .map_err(|e| anyhow::anyhow!("reading {}: {e}", dir.display()))?
315        .collect::<std::io::Result<Vec<_>>>()
316        .map_err(|e| anyhow::anyhow!("reading {}: {e}", dir.display()))?;
317    entries.sort_by_key(|entry| entry.file_name());
318    Ok(entries)
319}
320
321#[cfg(test)]
322mod tests;
pacenotes_server/store/generate.rs

pacenotes_server/store/
generate.rs
