diff --git a/microservices/semantic/requirements.txt b/microservices/semantic/requirements.txt new file mode 100644 index 000000000..bcc7ea8e0 --- /dev/null +++ b/microservices/semantic/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.0 +uvicorn==0.30.0 +sentence-transformers==3.0.1 +Pillow>=11.0.0 +numpy==1.26.4 +requests==2.32.3 +python-multipart==0.0.12 diff --git a/microservices/semantic/server.py b/microservices/semantic/server.py new file mode 100644 index 000000000..10e7fcabf --- /dev/null +++ b/microservices/semantic/server.py @@ -0,0 +1,102 @@ +import requests +from contextlib import asynccontextmanager +from io import BytesIO +from PIL import Image +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel, Field +from sentence_transformers import SentenceTransformer + +ARWEAVE_GATEWAY = "https://arweave.net" +ARWEAVE_ID_PATTERN = r"^[A-Za-z0-9_-]{43}$" +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 + +model = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global model + print("Loading CLIP model...") + model = SentenceTransformer("clip-ViT-B-32") + print("Model loaded.") + yield + + +app = FastAPI(title="NFT Embedding Server", lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +class EmbedTextRequest(BaseModel): + query: str + + +class EmbedArweaveRequest(BaseModel): + arweave_id: str = Field(pattern=ARWEAVE_ID_PATTERN) + + +def _decode_image(data: bytes) -> Image.Image: + try: + return Image.open(BytesIO(data)).convert("RGB") + except Exception: + raise HTTPException(400, "content is not a valid image") + + +@app.post("/embed/text") +def embed_text(req: EmbedTextRequest): + """Embed a text query into CLIP vector space.""" + embedding = model.encode(req.query) + return {"embedding": embedding.tolist()} + + +@app.post("/embed/image") +async def embed_image(file: UploadFile = File(...)): + """Embed a user-uploaded image. PIL decode is the real image check — + the client-supplied content-type header is not trusted.""" + data = await file.read() + if len(data) > MAX_UPLOAD_BYTES: + raise HTTPException(413, f"image exceeds {MAX_UPLOAD_BYTES} bytes") + + img = _decode_image(data) + embedding = model.encode(img) + return {"embedding": embedding.tolist()} + + +@app.post("/embed/arweave") +def embed_arweave(req: EmbedArweaveRequest): + """Verify an Arweave transaction is an image, then CLIP-encode it. + Two checks because the gateway is untrusted: HEAD content-type filters + cheaply, then PIL decode rejects bytes that don't actually parse.""" + url = f"{ARWEAVE_GATEWAY}/{req.arweave_id}" + + try: + head = requests.head(url, timeout=10, allow_redirects=True) + head.raise_for_status() + except requests.RequestException as e: + raise HTTPException(502, f"Arweave gateway unreachable: {e}") + + content_type = head.headers.get("content-type", "") + if not content_type.startswith("image/"): + raise HTTPException( + 400, + f"{req.arweave_id} is not an image (content-type: {content_type or 'unknown'})", + ) + + try: + resp = requests.get(url, timeout=30) + resp.raise_for_status() + except requests.RequestException as e: + raise HTTPException(502, f"Failed to download image: {e}") + + img = _decode_image(resp.content) + embedding = model.encode(img) + return { + "arweave_id": req.arweave_id, + "embedding": embedding.tolist(), + } diff --git a/src/alex_frontend/core/features/nft/components/SimilarNfts.tsx b/src/alex_frontend/core/features/nft/components/SimilarNfts.tsx new file mode 100644 index 000000000..e3e714d1c --- /dev/null +++ b/src/alex_frontend/core/features/nft/components/SimilarNfts.tsx @@ -0,0 +1,25 @@ +import React from "react"; +import { useSimilarNfts } from "../hooks/useSimilarNfts"; +import { useAppSelector } from "@/store/hooks/useAppSelector"; +import NftProvider from "@/components/NftProvider"; +import { NFTCard } from "@/features/nft"; +import type { AlexandrianToken } from "@/features/alexandrian/types"; + +export default function SimilarNfts({ arweaveId }: { arweaveId: string }) { + const { data: tokens, isLoading, error } = useSimilarNfts(arweaveId); + const { safe } = useAppSelector((state) => state.alexandrian); + + if (error) return null; + if (!isLoading && (!tokens || tokens.length === 0)) return null; + + return ( +
+

Similar NFTs

+ + {(token: AlexandrianToken) => ( + + )} + +
+ ); +} diff --git a/src/alex_frontend/core/features/nft/hooks/useSimilarNfts.ts b/src/alex_frontend/core/features/nft/hooks/useSimilarNfts.ts new file mode 100644 index 000000000..fe9aa555e --- /dev/null +++ b/src/alex_frontend/core/features/nft/hooks/useSimilarNfts.ts @@ -0,0 +1,77 @@ +import { useQuery } from "@tanstack/react-query"; +import { useAlexBackend } from "@/hooks/actors"; +import { arweaveIdToNat } from "@/utils/id_convert"; +import { createTokenAdapter } from "@/features/alexandrian/adapters/TokenAdapter"; +import type { AlexandrianToken } from "@/features/alexandrian/types"; + +const EMBEDDING_SERVER = + process.env.REACT_APP_EMBEDDING_SERVER || "https://lbry.youthumber.com"; + +const TOP_K = 20; + +interface SimilarityHit { + arweave_id: string; + score: number; +} + +export function useSimilarNfts(arweaveId: string | null) { + const { actor } = useAlexBackend(); + + return useQuery({ + queryKey: ["similar-nfts", arweaveId], + queryFn: async (): Promise => { + const id = arweaveId!; + + // Fast path: canister already has an embedding for this ID. + const indexed = await actor!.search_similar(id, TOP_K); + let hits: SimilarityHit[]; + + if ("Ok" in indexed) { + hits = indexed.Ok; + } else { + // Fallback: ask the server to verify+embed the Arweave image, + // then cosine-search the canister by the returned vector. + const res = await fetch(`${EMBEDDING_SERVER}/embed/arweave`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ arweave_id: id }), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({})); + throw new Error(body.detail ?? `embed/arweave ${res.status}`); + } + const { embedding } = await res.json(); + const vecRes = await actor!.search_by_vector(embedding, TOP_K); + if (!("Ok" in vecRes)) throw new Error(vecRes.Err); + hits = vecRes.Ok; + } + + const adapter = createTokenAdapter("NFT"); + return Promise.all( + hits.map(async (r) => { + const tokenId = arweaveIdToNat(r.arweave_id); + const ownerRes = await adapter.getOwnerOf([tokenId]); + const owner = ownerRes?.[0]?.[0]?.owner?.toString() || ""; + const icpInfo = await adapter.tokenToIcpInfo(tokenId); + return { + id: tokenId.toString(), + arweaveId: r.arweave_id, + owner, + collection: "NFT" as const, + ...icpInfo, + } as AlexandrianToken; + }), + ); + }, + enabled: !!actor && !!arweaveId, + staleTime: Infinity, + gcTime: 60 * 60 * 1000, + refetchOnWindowFocus: false, + refetchOnReconnect: false, + retry: (n, err) => { + const msg = (err as Error).message?.toLowerCase() ?? ""; + if (msg.includes("not an image") || msg.includes("not a valid image")) return false; + return n < 1; + }, + }); +} diff --git a/src/alex_frontend/core/features/perpetua/components/ShelfDetail.tsx b/src/alex_frontend/core/features/perpetua/components/ShelfDetail.tsx index 5594e3899..2c2b4be8a 100644 --- a/src/alex_frontend/core/features/perpetua/components/ShelfDetail.tsx +++ b/src/alex_frontend/core/features/perpetua/components/ShelfDetail.tsx @@ -14,7 +14,7 @@ import { useSortable, } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; -import { ArrowLeft, BookOpen, Copy, Check, GripVertical, Info, Layers, LayoutGrid, List, Loader2, Package } from "lucide-react"; +import { ArrowLeft, BookOpen, Copy, Check, Download, GripVertical, Info, Layers, LayoutGrid, List, Loader2, Package } from "lucide-react"; import { Button } from "@/lib/components/button"; import { Badge } from "@/lib/components/badge"; import { Skeleton } from "@/lib/components/skeleton"; @@ -29,6 +29,7 @@ import { useSetItemOrder } from "../hooks/useMutations"; import { useUsername } from "@/hooks/useUsername"; import { convertTimestamp } from "@/utils/general"; import { shortenPrincipal, getItemContentValue } from "../utils"; +import { natToArweaveId } from "@/utils/id_convert"; import type { Item } from "../types"; import MarkdownRenderer from "@/components/MarkdownRenderer"; import ItemCard from "./ItemCard"; @@ -321,6 +322,33 @@ export default function ShelfDetail({ shelfId, userId }: ShelfDetailProps) { <> + {contentFilter === "Nft" && filteredItems.length > 0 && ( + + )} )} diff --git a/src/alex_frontend/lbry/src/pages/NftPage.tsx b/src/alex_frontend/lbry/src/pages/NftPage.tsx index 6c1ced5c9..3ca8bbf94 100644 --- a/src/alex_frontend/lbry/src/pages/NftPage.tsx +++ b/src/alex_frontend/lbry/src/pages/NftPage.tsx @@ -8,6 +8,7 @@ import { NftContext } from "@/components/NftProvider"; import IcpInfo from "@/features/nft/components/Info/Icp"; import Tags from "@/features/nft/components/Info/Tags"; import Comment from "@/features/nft/components/Comment"; +import SimilarNfts from "@/features/nft/components/SimilarNfts"; import useTransactionMetadata from "@/features/nft/hooks/useTransactionMetadata"; import { AlexandrianToken } from "@/features/alexandrian/types"; import { natToArweaveId, arweaveIdToNat } from "@/utils/id_convert"; @@ -523,6 +524,9 @@ function NftPage() { + + {/* Row 2: Similar NFTs */} + );