Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 71 additions & 26 deletions apps/web/src/features/composer/Composer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
} from "../context-packet/context-packet.ts";
import {
admitAttachments,
materializeAttachmentCandidates,
toPromptAttachment,
type AttachmentCandidate,
type StagedAttachment,
Expand Down Expand Up @@ -71,11 +72,30 @@ const IMAGE_REVISION_REASON =
"Images cannot be added to a plan revision. Remove them or finish the revision first.";
const IMAGE_ACTIVE_TURN_REASON =
"Images can be sent with the next prompt after the running turn finishes.";
const IMAGE_PREPARATION_PENDING_REASON =
"Your selected images are still being prepared. Wait a moment and send again.";
const IMAGE_PREPARATION_FAILED_REASON =
"The selected images could not be prepared. Try choosing them again or selecting them through Files.";

function filesToCandidates(files: ArrayLike<File>): AttachmentCandidate[] {
return Array.from(files, (file) => ({ file }));
}

function attachmentIntakeState(sessionId: string): {
readonly existing: readonly StagedAttachment[];
readonly stagedBytes: number;
readonly stagedCount: number;
} {
const stagedBySession = composerStore.getState().attachmentsBySessionId;
let stagedBytes = 0;
let stagedCount = 0;
for (const staged of Object.values(stagedBySession)) {
stagedCount += staged.length;
for (const attachment of staged) stagedBytes += attachment.sizeBytes;
}
return { existing: stagedBySession[sessionId] ?? [], stagedBytes, stagedCount };
}

// ---------------------------------------------------------------------------
// Composer
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -153,6 +173,8 @@ export function Composer({
const contextNotice = useComposer((state) => state.contextNoticeBySessionId[sessionId] ?? null);
const textareaRef = useRef<HTMLTextAreaElement | null>(null);
const fileInputRef = useRef<HTMLInputElement | null>(null);
const attachmentIntakeTailRef = useRef<Promise<void>>(Promise.resolve());
const [preparingAttachmentCount, setPreparingAttachmentCount] = useState(0);
const [caret, setCaret] = useState(0);
const [menuIndex, setMenuIndex] = useState(0);
const [menuDismissed, setMenuDismissed] = useState(false);
Expand Down Expand Up @@ -330,6 +352,10 @@ export function Composer({
);

const submit = useCallback(() => {
if (preparingAttachmentCount > 0) {
setRejections([IMAGE_PREPARATION_PENDING_REASON]);
return;
}
const text = draft.trim();
if (text === "" && attachments.length === 0) return;
if (attachments.length > 0 && !controls.attachmentsSupported) {
Expand Down Expand Up @@ -373,6 +399,7 @@ export function Composer({
},
);
}, [
preparingAttachmentCount,
draft,
attachments,
controls.attachmentsSupported,
Expand All @@ -386,14 +413,18 @@ export function Composer({
]);

const queueFollowUp = useCallback(() => {
if (preparingAttachmentCount > 0) {
setRejections([IMAGE_PREPARATION_PENDING_REASON]);
return;
}
const text = draft.trim();
if (text === "") return;
if (attachments.length > 0) {
setRejections([IMAGE_ACTIVE_TURN_REASON]);
return;
}
runSubmission({ kind: "followUp", text }, { text: draft, attachmentIds: [] });
}, [draft, attachments.length, runSubmission]);
}, [draft, attachments.length, preparingAttachmentCount, runSubmission]);

// While observed/reconciling the gate carries its own reason; the generic
// host copy only applies when the host truly lacks image prompts.
Expand All @@ -404,26 +435,35 @@ export function Composer({
}, [attachmentsUnavailableReason, setRejections]);

const intake = useCallback(
(candidates: readonly AttachmentCandidate[]) => {
(candidates: readonly AttachmentCandidate[]): Promise<void> => {
if (!controls.attachmentsSupported) {
reportUnsupportedImages();
return;
}
const stagedBySession = composerStore.getState().attachmentsBySessionId;
const existing = stagedBySession[sessionId] ?? [];
let stagedBytes = 0;
let stagedCount = 0;
for (const staged of Object.values(stagedBySession)) {
stagedCount += staged.length;
for (const attachment of staged) stagedBytes += attachment.sizeBytes;
}
const result = admitAttachments(existing, candidates, { stagedBytes, stagedCount });
if (result.accepted.length > 0) {
composerStore.getState().addAttachments(sessionId, result.accepted);
return Promise.resolve();
}
setRejections(result.rejections);

setPreparingAttachmentCount((count) => count + 1);
const generation = composerStore.getState().sessionGeneration(sessionId);
// Serialize materialization as well as admission. This reserves the
// global staged byte/count budget before another picker batch allocates
// renderer-owned copies, bounding transient Android WebView memory.
const task = attachmentIntakeTailRef.current.then(async () => {
const initialState = attachmentIntakeState(sessionId);
const prepared = await materializeAttachmentCandidates(candidates, initialState);
if (composerStore.getState().sessionGeneration(sessionId) !== generation) return;
const { existing, stagedBytes, stagedCount } = attachmentIntakeState(sessionId);
const result = admitAttachments(existing, prepared.accepted, { stagedBytes, stagedCount });
if (result.accepted.length > 0) {
composerStore.getState().addAttachments(sessionId, result.accepted);
}
setRejections([...prepared.rejections, ...result.rejections]);
});
const settled = task
.catch(() => setRejections([IMAGE_PREPARATION_FAILED_REASON]))
.finally(() => setPreparingAttachmentCount((count) => Math.max(0, count - 1)));
attachmentIntakeTailRef.current = settled;
return settled;
},
[sessionId, controls.attachmentsSupported, reportUnsupportedImages],
[sessionId, controls.attachmentsSupported, reportUnsupportedImages, setRejections],
);

const requestAttachmentPicker = useCallback(() => {
Expand Down Expand Up @@ -494,7 +534,9 @@ export function Composer({
};

const primaryLabel = revisingPlanId !== null ? "Send revision" : turnActive ? "Steer" : "Send";
const canSubmit = !disabled && (draft.trim() !== "" || attachments.length > 0);
const preparingAttachments = preparingAttachmentCount > 0;
const canSubmit =
!disabled && !preparingAttachments && (draft.trim() !== "" || attachments.length > 0);
const runOptionsSummary = `${controls.modelLabel ?? "Host model"} · ${thinkingLabel(controls.thinking)}`;

return (
Expand Down Expand Up @@ -601,7 +643,7 @@ export function Composer({
if (event.dataTransfer.files.length === 0) return;
event.preventDefault();
if (disabled) return;
intake(filesToCandidates(event.dataTransfer.files));
void intake(filesToCandidates(event.dataTransfer.files));
}}
>
{revisingPlanId !== null && (
Expand Down Expand Up @@ -695,7 +737,7 @@ export function Composer({
if (event.clipboardData.files.length === 0) return;
event.preventDefault();
if (disabled) return;
intake(filesToCandidates(event.clipboardData.files));
void intake(filesToCandidates(event.clipboardData.files));
}}
onSelect={(event) => setCaret(event.currentTarget.selectionStart)}
placeholder={
Expand All @@ -717,8 +759,11 @@ export function Composer({
className="hidden"
multiple
onChange={(event) => {
if (event.target.files !== null) intake(filesToCandidates(event.target.files));
event.target.value = "";
const input = event.currentTarget;
if (input.files === null) return;
void intake(filesToCandidates(input.files)).finally(() => {
input.value = "";
});
}}
ref={fileInputRef}
type="file"
Expand Down Expand Up @@ -781,7 +826,7 @@ export function Composer({
</TooltipPopup>
</Tooltip>
<Button
disabled={disabled || sending || draft.trim() === ""}
disabled={disabled || sending || preparingAttachments || draft.trim() === ""}
onClick={queueFollowUp}
size="xs"
variant="outline"
Expand All @@ -791,7 +836,7 @@ export function Composer({
</>
)}
<Button
aria-busy={sending || undefined}
aria-busy={sending || preparingAttachments || undefined}
aria-label={primaryLabel}
className="ml-1"
disabled={!canSubmit || sending}
Expand Down Expand Up @@ -831,10 +876,10 @@ export function Composer({
onCancel={() => onIntent({ kind: "cancel" })}
onQueue={queueFollowUp}
onSubmit={submit}
primaryBusy={sending}
primaryBusy={sending || preparingAttachments}
primaryDisabled={!canSubmit || sending}
primaryLabel={primaryLabel}
queueDisabled={disabled || sending || draft.trim() === ""}
queueDisabled={disabled || sending || preparingAttachments || draft.trim() === ""}
turnActive={turnActive}
/>
</div>
Expand Down
115 changes: 115 additions & 0 deletions apps/web/src/features/composer/attachments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
// URLs stay renderer-local; the live runtime converts a File into an appserver
// upload and sends only the resulting image id across the wire.
import type { PromptAttachment } from "../session-runtime/intents.ts";
import {
readFileWithFileReader,
sniffPromptImageMimeType,
} from "../session-runtime/image-upload.ts";

export const MAX_ATTACHMENTS = 8;
export const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024; // Mirrors app-wire.
Expand Down Expand Up @@ -57,6 +61,25 @@ export interface AttachmentIntakeOptions {
readonly stagedCount?: number;
}

export interface AttachmentMaterialization {
readonly accepted: readonly AttachmentCandidate[];
readonly rejections: readonly string[];
}

export interface AttachmentMaterializationOptions {
/**
* Test seam. Production deliberately starts FileReader synchronously while
* the Android picker grant is fresh instead of retaining its lazy File.
*/
readonly readFile?: (file: File) => Promise<ArrayBuffer>;
/** Attachments already staged for the active session. */
readonly existing?: readonly StagedAttachment[];
/** Declared bytes already staged across every session. */
readonly stagedBytes?: number;
/** Images already staged across every session. */
readonly stagedCount?: number;
}

function formatBytes(bytes: number): string {
if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
if (bytes >= 1024) return `${Math.round(bytes / 1024)} kB`;
Expand All @@ -78,6 +101,98 @@ export function provisionalImageMediaType(file: File): string | null {
return IMAGE_EXTENSIONS[file.name.slice(dot + 1).toLowerCase()] ?? null;
}

/**
* Copy picker-backed Files into renderer-owned Files before the input is
* cleared. Android content providers can invalidate or mutate their synthetic
* File metadata after the change event; starting every FileReader here keeps
* the grant live, and staging only the immutable copies removes that lifetime
* from preview and submission.
*/
export async function materializeAttachmentCandidates(
candidates: readonly AttachmentCandidate[],
options: AttachmentMaterializationOptions = {},
): Promise<AttachmentMaterialization> {
const readFile = options.readFile ?? readFileWithFileReader;
const existing = options.existing ?? [];
let count = existing.length;
let stagedBytes =
options.stagedBytes ?? existing.reduce((total, attachment) => total + attachment.sizeBytes, 0);
let stagedCount = options.stagedCount ?? existing.length;
const seenFiles = new Set(existing.map((attachment) => attachment.file));
const results = await Promise.all(
candidates.map(async ({ file }) => {
const name = file.name || "untitled";
let rejection: string | null = null;
if (count >= MAX_ATTACHMENTS) {
rejection = `${name}: limit of ${MAX_ATTACHMENTS} attachments reached.`;
} else if (provisionalImageMediaType(file) === null) {
rejection = `${name}: attach a PNG, JPEG, WebP, or GIF image.`;
} else if (file.size === 0) {
rejection = `${name}: the image is empty.`;
} else if (file.size > MAX_ATTACHMENT_BYTES) {
rejection = `${name}: ${formatBytes(file.size)} is over the ${formatBytes(MAX_ATTACHMENT_BYTES)} limit.`;
} else if (seenFiles.has(file)) {
rejection = `${name}: already attached.`;
} else if (stagedCount >= MAX_STAGED_ATTACHMENTS) {
rejection = `${name}: the app already has ${MAX_STAGED_ATTACHMENTS} staged images. Remove one before adding another.`;
} else if (stagedBytes + file.size > MAX_STAGED_ATTACHMENT_BYTES) {
rejection = `${name}: staged images across sessions would exceed ${formatBytes(MAX_STAGED_ATTACHMENT_BYTES)}. Remove one before adding another.`;
}
if (rejection !== null) return { candidate: null, rejection } as const;

// Reserve the declared budget before starting another concurrent read.
// Final admission rechecks the owned File's actual byte size and current
// store state after every read settles.
count += 1;
stagedBytes += file.size;
stagedCount += 1;
seenFiles.add(file);

let buffer: ArrayBuffer;
try {
// This call occurs synchronously for every candidate before the outer
// function reaches its first await.
buffer = await readFile(file);
} catch {
return {
candidate: null,
rejection: `${file.name || "untitled"}: the selected image could not be read. Try choosing it again or selecting it through Files.`,
} as const;
}

const mediaType = sniffPromptImageMimeType(new Uint8Array(buffer));
if (mediaType === null) {
return {
candidate: null,
rejection: `${name}: attach a PNG, JPEG, WebP, or GIF image.`,
} as const;
}

try {
return {
candidate: {
file: new File([buffer], name, { type: mediaType }),
},
rejection: null,
} as const;
} catch {
return {
candidate: null,
rejection: `${name}: could not prepare a stable image copy.`,
} as const;
}
}),
);

const accepted: AttachmentCandidate[] = [];
const rejections: string[] = [];
for (const result of results) {
if (result.candidate === null) rejections.push(result.rejection);
else accepted.push(result.candidate);
}
return { accepted, rejections };
}

/**
* Validate candidates against the current attachment list. This first slice
* accepts images only; text/file parity waits for an explicit host protocol.
Expand Down
8 changes: 8 additions & 0 deletions apps/web/src/features/composer/composer-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ export interface ComposerStoreState {
finishSubmission(sessionId: string, token: SubmissionToken): void;
setSubmissionNotice(sessionId: string, notice: SubmissionNotice): void;
setAttachmentRejections(sessionId: string, rejections: readonly string[]): void;
/** Monotonic lifetime token invalidated by permanent session deletion. */
sessionGeneration(sessionId: string): number;
/** Invalidate async owners and release renderer-owned values after permanent deletion. */
disposeSession(sessionId: string): void;
}
Expand All @@ -62,6 +64,7 @@ export interface ComposerStoreOptions {
export function createComposerStore(options: ComposerStoreOptions = {}): ComposerStoreApi {
const revokePreviewUrl = options.revokePreviewUrl ?? ((url: string) => URL.revokeObjectURL(url));
let submissionSequence = 0;
const sessionGenerations = new Map<string, number>();
return createStore<ComposerStoreState>((set, get) => ({
attachmentsBySessionId: {},
contextItemsBySessionId: {},
Expand Down Expand Up @@ -212,7 +215,12 @@ export function createComposerStore(options: ComposerStoreOptions = {}): Compose
};
});
},
sessionGeneration: (sessionId) => sessionGenerations.get(sessionId) ?? 0,
disposeSession: (sessionId) => {
sessionGenerations.set(
sessionId,
(sessionGenerations.get(sessionId) ?? 0) + 1,
);
const attachments = get().attachmentsBySessionId[sessionId] ?? [];
for (const attachment of attachments) revokePreviewUrl(attachment.previewUrl);
set((state) => {
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/features/session-runtime/image-upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ function hex(bytes: Uint8Array): string {
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
}

function readFileWithFileReader(file: File): Promise<ArrayBuffer> {
export function readFileWithFileReader(file: File): Promise<ArrayBuffer> {
if (typeof globalThis.FileReader !== "function") {
return Promise.reject(new Error("This browser cannot read the selected file."));
}
Expand Down
Loading