Skip to content

Commit b33baa9

Browse files
aksOpsPaperclip-Paperclip
aksOps
and
Paperclip-Paperclip
committed
feat(intelligence): Phase 1 provenance model, repository identity, file inventory (RAN-146)
Implements the foundational contracts for the Repository Intelligence layer: - intelligence/ package: Provenance, RepositoryIdentity, FileInventory, FileEntry, FileClassification, CapabilityLevel, ArtifactManifest records - Provenance stored via prov_* keys in CodeNode.properties (round-trips through Neo4j) - RepositoryIdentity resolves git URL, commit SHA, branch from git CLI at analysis time - FileInventory builds a deterministic sorted list of all discovered files with classification heuristics (source/config/doc/test/generated) - GraphBuilder now accepts Provenance as constructor parameter (not a mutable setter) - Analyzer and EnrichCommand stamp provenance on all nodes during pipeline - BundleCommand upgraded to use ArtifactManifest record (repo identity, inventory summary) - Tests: ProvenanceTest (6), FileInventoryTest (8), ArtifactManifestTest (5), ProvenanceIntegrationTest (2) — all nodes carry provenance + determinism verified Addresses PE architecture review blocking constraints from RAN-150: - BLOCKING 1: Provenance uses properties map (prov_* prefix), not direct CodeNode fields - BLOCKING 2: Provenance is a GraphBuilder constructor parameter, not a setter - BLOCKING 3: FileEntry added to intelligence/ without modifying DiscoveredFile Co-Authored-By: Paperclip <noreply@paperclip.ing>
1 parent a912c6a commit b33baa9

17 files changed

Lines changed: 820 additions & 39 deletions

‎src/main/java/io/github/randomcodespace/iq/analyzer/Analyzer.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import io.github.randomcodespace.iq.analyzer.linker.Linker;
44
import io.github.randomcodespace.iq.cache.AnalysisCache;
55
import io.github.randomcodespace.iq.cache.FileHasher;
6+
import io.github.randomcodespace.iq.cli.VersionCommand;
67
import io.github.randomcodespace.iq.config.CodeIqConfig;
78
import io.github.randomcodespace.iq.config.ProjectConfig;
89
import io.github.randomcodespace.iq.config.ProjectConfigLoader;
@@ -12,6 +13,12 @@
1213
import io.github.randomcodespace.iq.detector.DetectorResult;
1314
import io.github.randomcodespace.iq.detector.DetectorUtils;
1415
import io.github.randomcodespace.iq.grammar.AntlrParserFactory;
16+
import io.github.randomcodespace.iq.intelligence.CapabilityLevel;
17+
import io.github.randomcodespace.iq.intelligence.FileClassification;
18+
import io.github.randomcodespace.iq.intelligence.FileEntry;
19+
import io.github.randomcodespace.iq.intelligence.FileInventory;
20+
import io.github.randomcodespace.iq.intelligence.Provenance;
21+
import io.github.randomcodespace.iq.intelligence.RepositoryIdentity;
1522
import io.github.randomcodespace.iq.model.CodeEdge;
1623
import io.github.randomcodespace.iq.model.CodeNode;
1724
import io.github.randomcodespace.iq.model.NodeKind;
@@ -227,6 +234,17 @@ private AnalysisResult runWithCache(Path root, Integer parallelism, AnalysisCach
227234
int totalFiles = files.size();
228235
report.accept("Found " + totalFiles + " files");
229236

237+
// 1b. Resolve repository identity and build file inventory
238+
RepositoryIdentity repoIdentity = RepositoryIdentity.resolve(root);
239+
FileInventory fileInventory = buildFileInventory(files);
240+
Provenance provenance = new Provenance(
241+
repoIdentity.repoUrl(),
242+
repoIdentity.commitSha(),
243+
VersionCommand.VERSION,
244+
Provenance.CURRENT_SCHEMA_VERSION,
245+
CapabilityLevel.PARTIAL
246+
);
247+
230248
// Compute language breakdown
231249
Map<String, Integer> languageBreakdown = new HashMap<>();
232250
for (DiscoveredFile f : files) {
@@ -300,7 +318,7 @@ private AnalysisResult runWithCache(Path root, Integer parallelism, AnalysisCach
300318

301319
// 3. Build graph (batched)
302320
report.accept("Building graph...");
303-
var builder = new GraphBuilder();
321+
var builder = new GraphBuilder(provenance);
304322
int filesAnalyzed = 0;
305323
for (int i = 0; i < resultSlots.length; i++) {
306324
DetectorResult result = resultSlots[i];
@@ -329,6 +347,7 @@ private AnalysisResult runWithCache(Path root, Integer parallelism, AnalysisCach
329347
String projectDirName = root.getFileName() != null ? root.getFileName().toString() : "root";
330348
var serviceResult = serviceDetector.detect(allNodes, builder.getEdges(), projectDirName, root);
331349
if (!serviceResult.serviceNodes().isEmpty()) {
350+
serviceResult.serviceNodes().forEach(n -> n.setProvenance(provenance));
332351
builder.addNodes(serviceResult.serviceNodes());
333352
builder.addEdges(serviceResult.serviceEdges());
334353
allNodes = builder.getNodes(); // refresh reference after adding service nodes
@@ -1257,6 +1276,20 @@ DetectorResult analyzeFile(DiscoveredFile file, Path repoPath, DetectorRegistry
12571276
return DetectorResult.of(allNodes, allEdges);
12581277
}
12591278

1279+
/**
1280+
* Build a deterministic FileInventory from the list of discovered files.
1281+
* Content hashes are not computed here (too expensive at discovery time); they remain null.
1282+
*/
1283+
private static FileInventory buildFileInventory(List<DiscoveredFile> files) {
1284+
List<FileEntry> entries = new ArrayList<>(files.size());
1285+
for (DiscoveredFile f : files) {
1286+
String relPath = f.path().toString().replace('\\', '/');
1287+
FileClassification cls = FileEntry.classify(relPath, f.language());
1288+
entries.add(new FileEntry(relPath, f.language(), f.sizeBytes(), null, cls));
1289+
}
1290+
return new FileInventory(entries);
1291+
}
1292+
12601293
/**
12611294
* Get the current git HEAD commit SHA, or null if not a git repo.
12621295
*/

‎src/main/java/io/github/randomcodespace/iq/analyzer/GraphBuilder.java‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import io.github.randomcodespace.iq.analyzer.linker.LinkResult;
44
import io.github.randomcodespace.iq.analyzer.linker.Linker;
55
import io.github.randomcodespace.iq.detector.DetectorResult;
6+
import io.github.randomcodespace.iq.intelligence.Provenance;
67
import io.github.randomcodespace.iq.model.CodeEdge;
78
import io.github.randomcodespace.iq.model.CodeNode;
89
import org.slf4j.Logger;
@@ -34,13 +35,23 @@ public class GraphBuilder {
3435
private final List<CodeEdge> deferredEdges = new ArrayList<>();
3536
private int droppedEdgeCount = 0;
3637
private final int batchSize;
38+
private final Provenance provenance;
3739

3840
public GraphBuilder() {
39-
this(1000);
41+
this(1000, null);
4042
}
4143

4244
public GraphBuilder(int batchSize) {
45+
this(batchSize, null);
46+
}
47+
48+
public GraphBuilder(Provenance provenance) {
49+
this(1000, provenance);
50+
}
51+
52+
public GraphBuilder(int batchSize, Provenance provenance) {
4353
this.batchSize = Math.max(1, batchSize);
54+
this.provenance = provenance;
4455
}
4556

4657
/**
@@ -67,11 +78,18 @@ public void addEdges(List<CodeEdge> edges) {
6778

6879
/**
6980
* Flush all buffered data: insert nodes first, then edges.
70-
* Edges whose source or target node doesn't exist are deferred.
81+
* Applies provenance to every node, then partitions edges into valid/deferred.
7182
*
7283
* @return a snapshot of all valid nodes and edges
7384
*/
7485
public FlushResult flush() {
86+
// Stamp provenance on every node
87+
if (provenance != null) {
88+
for (CodeNode node : allNodes) {
89+
node.setProvenance(provenance);
90+
}
91+
}
92+
7593
// Build the set of all node IDs
7694
Set<String> nodeIds = new HashSet<>();
7795
for (CodeNode node : allNodes) {

‎src/main/java/io/github/randomcodespace/iq/cli/BundleCommand.java‎

Lines changed: 24 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
import io.github.randomcodespace.iq.config.CodeIqConfig;
77
import io.github.randomcodespace.iq.flow.FlowEngine;
88
import io.github.randomcodespace.iq.graph.GraphStore;
9+
import io.github.randomcodespace.iq.intelligence.ArtifactManifest;
10+
import io.github.randomcodespace.iq.intelligence.FileInventory;
11+
import io.github.randomcodespace.iq.intelligence.RepositoryIdentity;
912
import org.springframework.beans.factory.annotation.Autowired;
1013
import org.springframework.stereotype.Component;
1114
import picocli.CommandLine.Command;
@@ -120,7 +123,6 @@ public Integer call() {
120123

121124
// Get node/edge counts from H2 cache
122125
long nodeCount = 0, edgeCount = 0;
123-
int filesAnalyzed = 0;
124126
if (Files.isDirectory(h2Dir)) {
125127
try (var cache = new AnalysisCache(h2Dir.resolve("analysis-cache.db"))) {
126128
nodeCount = cache.getNodeCount();
@@ -136,8 +138,9 @@ public Integer call() {
136138

137139
// 1. manifest.json
138140
CliOutput.info(" Writing manifest.json");
139-
String manifest = createManifest(projectName, bundleTag, version,
140-
nodeCount, edgeCount, filesAnalyzed, !noSource, includeJar);
141+
RepositoryIdentity repoIdentity = RepositoryIdentity.resolve(root);
142+
String manifest = createManifest(projectName, bundleTag, version, repoIdentity,
143+
nodeCount, edgeCount, !noSource, includeJar);
141144
writeEntry(zos, "manifest.json", manifest);
142145

143146
// 2. serve.sh
@@ -217,27 +220,27 @@ public Integer call() {
217220
// --- Manifest ---
218221

219222
private String createManifest(String projectName, String bundleTag, String version,
220-
long nodeCount, long edgeCount, int filesAnalyzed,
223+
RepositoryIdentity repoIdentity,
224+
long nodeCount, long edgeCount,
221225
boolean includesSource, boolean includesJar) {
222-
Map<String, Object> m = new LinkedHashMap<>();
223-
m.put("bundle_format", 2);
224-
m.put("tag", bundleTag);
225-
m.put("project", projectName);
226-
m.put("osscodeiq_version", version);
227-
m.put("created_at", Instant.now().toString());
228-
m.put("backend", "neo4j");
229-
m.put("node_count", nodeCount);
230-
m.put("edge_count", edgeCount);
231-
m.put("files_analyzed", filesAnalyzed);
232-
m.put("includes_source", includesSource);
233-
m.put("includes_jar", includesJar);
234-
235-
String gitSha = getGitSha();
236-
if (gitSha != null) m.put("git_sha", gitSha);
237-
226+
var manifest = new ArtifactManifest(
227+
ArtifactManifest.BUNDLE_FORMAT_VERSION,
228+
bundleTag,
229+
projectName,
230+
version,
231+
io.github.randomcodespace.iq.intelligence.Provenance.CURRENT_SCHEMA_VERSION,
232+
Instant.now().toString(),
233+
repoIdentity,
234+
FileInventory.EMPTY.toSummary(),
235+
nodeCount,
236+
edgeCount,
237+
includesSource,
238+
includesJar,
239+
null
240+
);
238241
try {
239242
return new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)
240-
.writeValueAsString(m);
243+
.writeValueAsString(manifest.toMap());
241244
} catch (Exception e) {
242245
return "{}";
243246
}
@@ -464,17 +467,4 @@ private void writeEntry(ZipOutputStream zos, String name, String content, String
464467
writeEntry(zos, name, content);
465468
}
466469

467-
private String getGitSha() {
468-
try {
469-
ProcessBuilder pb = new ProcessBuilder("git", "rev-parse", "HEAD")
470-
.directory(path.toAbsolutePath().normalize().toFile())
471-
.redirectErrorStream(true);
472-
Process proc = pb.start();
473-
String sha = new String(proc.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim();
474-
int exitCode = proc.waitFor();
475-
return (exitCode == 0 && sha.length() >= 7) ? sha : null;
476-
} catch (Exception ignored) {
477-
return null;
478-
}
479-
}
480470
}

‎src/main/java/io/github/randomcodespace/iq/cli/EnrichCommand.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
import io.github.randomcodespace.iq.analyzer.linker.Linker;
66
import io.github.randomcodespace.iq.cache.AnalysisCache;
77
import io.github.randomcodespace.iq.config.CodeIqConfig;
8+
import io.github.randomcodespace.iq.intelligence.CapabilityLevel;
9+
import io.github.randomcodespace.iq.intelligence.Provenance;
10+
import io.github.randomcodespace.iq.intelligence.RepositoryIdentity;
811
import io.github.randomcodespace.iq.model.CodeEdge;
912
import io.github.randomcodespace.iq.model.CodeNode;
1013
import io.github.randomcodespace.iq.model.EdgeKind;
@@ -116,7 +119,15 @@ private int enrichFromCache(AnalysisCache cache, Path root, NumberFormat nf, Ins
116119

117120
// 2. Run linkers (these work on in-memory node/edge lists)
118121
CliOutput.step("\uD83D\uDD17", "Running cross-file linkers...");
119-
var builder = new GraphBuilder();
122+
RepositoryIdentity repoIdentity = RepositoryIdentity.resolve(root);
123+
Provenance provenance = new Provenance(
124+
repoIdentity.repoUrl(),
125+
repoIdentity.commitSha(),
126+
VersionCommand.VERSION,
127+
Provenance.CURRENT_SCHEMA_VERSION,
128+
CapabilityLevel.PARTIAL
129+
);
130+
var builder = new GraphBuilder(provenance);
120131
for (CodeNode node : allNodes) {
121132
builder.addNodes(List.of(node));
122133
}
@@ -147,6 +158,7 @@ private int enrichFromCache(AnalysisCache cache, Path root, NumberFormat nf, Ins
147158
String projectName = root.getFileName().toString();
148159
var serviceResult = serviceDetector.detect(enrichedNodes, enrichedEdges, projectName, root);
149160
if (!serviceResult.serviceNodes().isEmpty()) {
161+
serviceResult.serviceNodes().forEach(n -> n.setProvenance(provenance));
150162
// Add service nodes and edges to the builder
151163
builder.addNodes(serviceResult.serviceNodes());
152164
builder.addEdges(serviceResult.serviceEdges());

‎src/main/java/io/github/randomcodespace/iq/cli/VersionCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
description = "Show version info")
1717
public class VersionCommand implements Callable<Integer> {
1818

19-
static final String VERSION = "0.1.0-SNAPSHOT";
19+
public static final String VERSION = "0.1.0-SNAPSHOT";
2020

2121
private final DetectorRegistry registry;
2222

‎src/main/java/io/github/randomcodespace/iq/intelligence/ArtifactManifest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package io.github.randomcodespace.iq.intelligence;
2+
3+
import java.util.Map;
4+
5+
/**
6+
* Artifact manifest — extends the bundle manifest with repository identity,
7+
* schema version, extractor version, file inventory summary, and integrity checksums.
8+
*
9+
* @param bundleFormat Always {@code 2} for this record.
10+
* @param tag User-supplied bundle tag (may be null).
11+
* @param project Project name.
12+
* @param extractorVersion Version of the code-iq extractor that built this bundle.
13+
* @param schemaVersion Graph schema version at bundle time.
14+
* @param createdAt ISO-8601 timestamp.
15+
* @param repositoryIdentity Git/VCS identity of the analysed repo.
16+
* @param fileInventorySummary Summary from {@link FileInventory#toSummary()}.
17+
* @param nodeCount Total graph nodes.
18+
* @param edgeCount Total graph edges.
19+
* @param includesSource Whether source files are bundled.
20+
* @param includesJar Whether the CLI JAR is bundled.
21+
* @param checksums SHA-256 digests of key bundle entries (entry → hex digest).
22+
*/
23+
public record ArtifactManifest(
24+
int bundleFormat,
25+
String tag,
26+
String project,
27+
String extractorVersion,
28+
int schemaVersion,
29+
String createdAt,
30+
RepositoryIdentity repositoryIdentity,
31+
Map<String, Object> fileInventorySummary,
32+
long nodeCount,
33+
long edgeCount,
34+
boolean includesSource,
35+
boolean includesJar,
36+
Map<String, String> checksums
37+
) {
38+
public static final int BUNDLE_FORMAT_VERSION = 2;
39+
40+
/**
41+
* Serialise to a JSON-friendly {@link Map} (preserves insertion order).
42+
* Null/empty fields are omitted for a clean manifest.
43+
*/
44+
public Map<String, Object> toMap() {
45+
var m = new java.util.LinkedHashMap<String, Object>();
46+
m.put("bundle_format", bundleFormat);
47+
if (tag != null) m.put("tag", tag);
48+
m.put("project", project);
49+
m.put("extractor_version", extractorVersion);
50+
m.put("schema_version", schemaVersion);
51+
m.put("created_at", createdAt);
52+
53+
// Repository identity
54+
if (repositoryIdentity != null) {
55+
var ri = new java.util.LinkedHashMap<String, Object>();
56+
if (repositoryIdentity.repoUrl() != null) ri.put("repo_url", repositoryIdentity.repoUrl());
57+
if (repositoryIdentity.commitSha() != null) ri.put("commit_sha", repositoryIdentity.commitSha());
58+
if (repositoryIdentity.branch() != null) ri.put("branch", repositoryIdentity.branch());
59+
if (repositoryIdentity.buildTimestamp() != null)
60+
ri.put("build_timestamp", repositoryIdentity.buildTimestamp().toString());
61+
if (!ri.isEmpty()) m.put("repository", ri);
62+
}
63+
64+
// File inventory summary
65+
if (fileInventorySummary != null && !fileInventorySummary.isEmpty()) {
66+
m.put("file_inventory", fileInventorySummary);
67+
}
68+
69+
m.put("backend", "neo4j");
70+
m.put("node_count", nodeCount);
71+
m.put("edge_count", edgeCount);
72+
m.put("includes_source", includesSource);
73+
m.put("includes_jar", includesJar);
74+
75+
if (checksums != null && !checksums.isEmpty()) {
76+
m.put("checksums", checksums);
77+
}
78+
return m;
79+
}
80+
}

‎src/main/java/io/github/randomcodespace/iq/intelligence/CapabilityLevel.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package io.github.randomcodespace.iq.intelligence;
2+
3+
/**
4+
* Confidence level for intelligence capabilities on a given language or feature.
5+
* Used in provenance records and capability matrix entries.
6+
*/
7+
public enum CapabilityLevel {
8+
/** Full semantic understanding — AST-level, cross-file, high confidence. */
9+
EXACT,
10+
/** Partial coverage — some constructs detected, others may be missed. */
11+
PARTIAL,
12+
/** Lexical/text search only — no structural analysis. */
13+
LEXICAL_ONLY,
14+
/** Language or feature is not supported by current extractors. */
15+
UNSUPPORTED
16+
}

‎src/main/java/io/github/randomcodespace/iq/intelligence/FileClassification.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package io.github.randomcodespace.iq.intelligence;
2+
3+
/**
4+
* Heuristic classification of a file's role in the repository.
5+
* Determined by file extension and path conventions.
6+
*/
7+
public enum FileClassification {
8+
/** Production source code. */
9+
SOURCE,
10+
/** Configuration files (YAML, JSON, TOML, properties, etc.). */
11+
CONFIG,
12+
/** Documentation (Markdown, AsciiDoc, plain text, etc.). */
13+
DOC,
14+
/** Test code (paths containing test/, spec/, __tests__, etc.). */
15+
TEST,
16+
/** Generated code (paths containing generated/, build/, target/, etc.). */
17+
GENERATED
18+
}

0 commit comments

Comments
 (0)