diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 122819d..97f2a27 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -2130,6 +2130,46 @@ "installPolicy": "AVAILABLE", "authPolicy": "ON_INSTALL", "category": "Lifestyle" + }, + { + "name": "cloudflare", + "source": { + "source": "local", + "path": "./plugins/cloudflare" + }, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_INSTALL", + "category": "Coding" + }, + { + "name": "github", + "source": { + "source": "local", + "path": "./plugins/github" + }, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_INSTALL", + "category": "Coding" + }, + { + "name": "notion", + "source": { + "source": "local", + "path": "./plugins/notion" + }, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_INSTALL", + "category": "Productivity" + }, + { + "name": "sentry", + "source": { + "source": "local", + "path": "./plugins/sentry" + }, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_INSTALL", + "category": "Coding" } ] } diff --git a/plugins/cloudflare/.codex-plugin/plugin.json b/plugins/cloudflare/.codex-plugin/plugin.json new file mode 100644 index 0000000..ec1a244 --- /dev/null +++ b/plugins/cloudflare/.codex-plugin/plugin.json @@ -0,0 +1,41 @@ +{ + "name": "cloudflare", + "version": "0.1.0", + "description": "Use Cloudflare platform skills for deployment, infrastructure, and analytics workflows.", + "author": { + "name": "OpenAI", + "email": "support@openai.com", + "url": "https://openai.com/" + }, + "homepage": "https://www.cloudflare.com/", + "repository": "https://github.com/openai/plugins", + "license": "MIT", + "keywords": [ + "cloudflare", + "deployment", + "edge", + "analytics", + "wrangler", + "agents-sdk" + ], + "skills": "./skills/", + "interface": { + "displayName": "Cloudflare", + "shortDescription": "Use Cloudflare platform skills", + "longDescription": "Use Cloudflare skills to choose the right product, verify setup, and work across Workers, Pages, Wrangler, Agents SDK, storage, AI, networking, security, and analytics services.", + "developerName": "OpenAI", + "category": "Productivity", + "capabilities": [ + "Interactive", + "Write" + ], + "websiteURL": "https://www.cloudflare.com/", + "privacyPolicyURL": "https://www.cloudflare.com/privacypolicy/", + "termsOfServiceURL": "https://www.cloudflare.com/website-terms/", + "defaultPrompt": "Use Cloudflare context to choose the right product, verify setup, or plan the right deployment, data, or analytics workflow without guessing infrastructure.", + "composerIcon": "./assets/cloudflare-small.svg", + "logo": "./assets/cloudflare.png", + "screenshots": [], + "brandColor": "#F48120" + } +} diff --git a/plugins/cloudflare/assets/cloudflare-small.svg b/plugins/cloudflare/assets/cloudflare-small.svg new file mode 100644 index 0000000..7feedbc --- /dev/null +++ b/plugins/cloudflare/assets/cloudflare-small.svg @@ -0,0 +1,3 @@ + diff --git a/plugins/cloudflare/assets/cloudflare.png b/plugins/cloudflare/assets/cloudflare.png new file mode 100644 index 0000000..cf23088 Binary files /dev/null and b/plugins/cloudflare/assets/cloudflare.png differ diff --git a/plugins/cloudflare/skills/agents-sdk/LICENSE.txt b/plugins/cloudflare/skills/agents-sdk/LICENSE.txt new file mode 100644 index 0000000..13e25df --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of + any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don\'t include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/plugins/cloudflare/skills/agents-sdk/SKILL.md b/plugins/cloudflare/skills/agents-sdk/SKILL.md new file mode 100644 index 0000000..846e882 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/SKILL.md @@ -0,0 +1,155 @@ +--- +name: agents-sdk +description: Build AI agents on Cloudflare Workers using the Agents SDK. Load when creating stateful agents, durable workflows, real-time WebSocket apps, scheduled tasks, MCP servers, or chat applications. Covers Agent class, state management, callable RPC, Workflows integration, and React hooks. Biases towards retrieval from Cloudflare docs over pre-trained knowledge. +--- + +# Cloudflare Agents SDK + +Your knowledge of the Agents SDK may be outdated. **Prefer retrieval over pre-training** for any Agents SDK task. + +## Retrieval Sources + +Fetch current docs from `https://github.com/cloudflare/agents/tree/main/docs` before implementing. + +| Topic | Doc | Use for | +|-------|-----|---------| +| Getting started | `docs/getting-started.md` | First agent, project setup | +| State | `docs/state.md` | `setState`, `validateStateChange`, persistence | +| Routing | `docs/routing.md` | URL patterns, `routeAgentRequest`, `basePath` | +| Callable methods | `docs/callable-methods.md` | `@callable`, RPC, streaming, timeouts | +| Scheduling | `docs/scheduling.md` | `schedule()`, `scheduleEvery()`, cron | +| Workflows | `docs/workflows.md` | `AgentWorkflow`, durable multi-step tasks | +| HTTP/WebSockets | `docs/http-websockets.md` | Lifecycle hooks, hibernation | +| Email | `docs/email.md` | Email routing, secure reply resolver | +| MCP client | `docs/mcp-client.md` | Connecting to MCP servers | +| MCP server | `docs/mcp-servers.md` | Building MCP servers with `McpAgent` | +| Client SDK | `docs/client-sdk.md` | `useAgent`, `useAgentChat`, React hooks | +| Human-in-the-loop | `docs/human-in-the-loop.md` | Approval flows, pausing workflows | +| Resumable streaming | `docs/resumable-streaming.md` | Stream recovery on disconnect | + +Cloudflare docs: https://developers.cloudflare.com/agents/ + +## Capabilities + +The Agents SDK provides: + +- **Persistent state** - SQLite-backed, auto-synced to clients +- **Callable RPC** - `@callable()` methods invoked over WebSocket +- **Scheduling** - One-time, recurring (`scheduleEvery`), and cron tasks +- **Workflows** - Durable multi-step background processing via `AgentWorkflow` +- **MCP integration** - Connect to MCP servers or build your own with `McpAgent` +- **Email handling** - Receive and reply to emails with secure routing +- **Streaming chat** - `AIChatAgent` with resumable streams +- **React hooks** - `useAgent`, `useAgentChat` for client apps + +## FIRST: Verify Installation + +```bash +npm ls agents # Should show agents package +``` + +If not installed: +```bash +npm install agents +``` + +## Wrangler Configuration + +```jsonc +{ + "durable_objects": { + "bindings": [{ "name": "MyAgent", "class_name": "MyAgent" }] + }, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["MyAgent"] }] +} +``` + +## Agent Class + +```typescript +import { Agent, routeAgentRequest, callable } from "agents"; + +type State = { count: number }; + +export class Counter extends Agent { + initialState = { count: 0 }; + + // Validation hook - runs before state persists (sync, throwing rejects the update) + validateStateChange(nextState: State, source: Connection | "server") { + if (nextState.count < 0) throw new Error("Count cannot be negative"); + } + + // Notification hook - runs after state persists (async, non-blocking) + onStateUpdate(state: State, source: Connection | "server") { + console.log("State updated:", state); + } + + @callable() + increment() { + this.setState({ count: this.state.count + 1 }); + return this.state.count; + } +} + +export default { + fetch: (req, env) => routeAgentRequest(req, env) ?? new Response("Not found", { status: 404 }) +}; +``` + +## Routing + +Requests route to `/agents/{agent-name}/{instance-name}`: + +| Class | URL | +|-------|-----| +| `Counter` | `/agents/counter/user-123` | +| `ChatRoom` | `/agents/chat-room/lobby` | + +Client: `useAgent({ agent: "Counter", name: "user-123" })` + +## Core APIs + +| Task | API | +|------|-----| +| Read state | `this.state.count` | +| Write state | `this.setState({ count: 1 })` | +| SQL query | `` this.sql`SELECT * FROM users WHERE id = ${id}` `` | +| Schedule (delay) | `await this.schedule(60, "task", payload)` | +| Schedule (cron) | `await this.schedule("0 * * * *", "task", payload)` | +| Schedule (interval) | `await this.scheduleEvery(30, "poll")` | +| RPC method | `@callable() myMethod() { ... }` | +| Streaming RPC | `@callable({ streaming: true }) stream(res) { ... }` | +| Start workflow | `await this.runWorkflow("ProcessingWorkflow", params)` | + +## React Client + +```tsx +import { useAgent } from "agents/react"; + +function App() { + const [state, setLocalState] = useState({ count: 0 }); + + const agent = useAgent({ + agent: "Counter", + name: "my-instance", + onStateUpdate: (newState) => setLocalState(newState), + onIdentity: (name, agentType) => console.log(`Connected to ${name}`) + }); + + return ( + + ); +} +``` + +## References + +- **[references/workflows.md](references/workflows.md)** - Durable Workflows integration +- **[references/callable.md](references/callable.md)** - RPC methods, streaming, timeouts +- **[references/state-scheduling.md](references/state-scheduling.md)** - State persistence, scheduling +- **[references/streaming-chat.md](references/streaming-chat.md)** - AIChatAgent, resumable streams +- **[references/mcp.md](references/mcp.md)** - MCP server integration +- **[references/email.md](references/email.md)** - Email routing and handling +- **[references/codemode.md](references/codemode.md)** - Code Mode (experimental) diff --git a/plugins/cloudflare/skills/agents-sdk/agents/openai.yaml b/plugins/cloudflare/skills/agents-sdk/agents/openai.yaml new file mode 100644 index 0000000..6696289 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Cloudflare Agents SDK" + short_description: "Build stateful agents on Cloudflare Workers" + icon_small: "./assets/cloudflare-small.svg" + icon_large: "./assets/cloudflare.png" + default_prompt: "Use the Cloudflare Agents SDK to design or implement this agent on Workers with the right state, routing, and workflow patterns." diff --git a/plugins/cloudflare/skills/agents-sdk/assets/cloudflare-small.svg b/plugins/cloudflare/skills/agents-sdk/assets/cloudflare-small.svg new file mode 100644 index 0000000..7feedbc --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/assets/cloudflare-small.svg @@ -0,0 +1,3 @@ + diff --git a/plugins/cloudflare/skills/agents-sdk/assets/cloudflare.png b/plugins/cloudflare/skills/agents-sdk/assets/cloudflare.png new file mode 100644 index 0000000..cf23088 Binary files /dev/null and b/plugins/cloudflare/skills/agents-sdk/assets/cloudflare.png differ diff --git a/plugins/cloudflare/skills/agents-sdk/references/callable.md b/plugins/cloudflare/skills/agents-sdk/references/callable.md new file mode 100644 index 0000000..241d30c --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/callable.md @@ -0,0 +1,92 @@ +# Callable Methods + +Fetch `docs/callable-methods.md` from `https://github.com/cloudflare/agents/tree/main/docs` for complete documentation. + +## Overview + +`@callable()` exposes agent methods to clients via WebSocket RPC. + +```typescript +import { Agent, callable } from "agents"; + +export class MyAgent extends Agent { + @callable() + async greet(name: string): Promise { + return `Hello, ${name}!`; + } + + @callable() + async processData(data: unknown): Promise { + // Long-running work + return result; + } +} +``` + +## Client Usage + +```typescript +// Basic call +const greeting = await agent.call("greet", ["World"]); + +// With timeout +const result = await agent.call("processData", [data], { + timeout: 5000 // 5 second timeout +}); +``` + +## Streaming Responses + +```typescript +import { Agent, callable, StreamingResponse } from "agents"; + +export class MyAgent extends Agent { + @callable({ streaming: true }) + async streamResults(stream: StreamingResponse, query: string) { + for await (const item of fetchResults(query)) { + stream.send(JSON.stringify(item)); + } + stream.close(); + } + + @callable({ streaming: true }) + async streamWithError(stream: StreamingResponse) { + try { + // ... work + } catch (error) { + stream.error(error.message); // Signal error to client + return; + } + stream.close(); + } +} +``` + +Client with streaming: + +```typescript +await agent.call("streamResults", ["search term"], { + stream: { + onChunk: (data) => console.log("Chunk:", data), + onDone: () => console.log("Complete"), + onError: (error) => console.error("Error:", error) + } +}); +``` + +## Introspection + +```typescript +// Get list of callable methods on an agent +const methods = await agent.call("getCallableMethods", []); +// Returns: ["greet", "processData", "streamResults", ...] +``` + +## When to Use + +| Scenario | Use | +|----------|-----| +| Browser/mobile calling agent | `@callable()` | +| External service calling agent | `@callable()` | +| Worker calling agent (same codebase) | DO RPC directly | +| Agent calling another agent | `getAgentByName()` + DO RPC | diff --git a/plugins/cloudflare/skills/agents-sdk/references/codemode.md b/plugins/cloudflare/skills/agents-sdk/references/codemode.md new file mode 100644 index 0000000..f95ab38 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/codemode.md @@ -0,0 +1,207 @@ +# Code Mode (Experimental) + +Code Mode generates executable JavaScript instead of making individual tool calls. This significantly reduces token usage and enables complex multi-tool workflows. + +## Why Code Mode? + +Traditional tool calling: +- One tool call per LLM request +- Multiple round-trips for chained operations +- High token usage for complex workflows + +Code Mode: +- LLM generates code that orchestrates multiple tools +- Single execution for complex workflows +- Self-debugging and error recovery +- Ideal for MCP server orchestration + +## Setup + +### 1. Wrangler Config + +```jsonc +{ + "name": "my-agent-worker", + "compatibility_flags": ["experimental", "enable_ctx_exports"], + "durable_objects": { + // "class_name" must match your Agent class name exactly + "bindings": [{ "name": "MyAgent", "class_name": "MyAgent" }] + }, + "migrations": [ + // Required: list all Agent classes for SQLite storage + { "tag": "v1", "new_sqlite_classes": ["MyAgent"] } + ], + "services": [ + { + "binding": "globalOutbound", + // "service" must match "name" above + "service": "my-agent-worker", + "entrypoint": "globalOutbound" + }, + { + "binding": "CodeModeProxy", + "service": "my-agent-worker", + "entrypoint": "CodeModeProxy" + } + ], + "worker_loaders": [{ "binding": "LOADER" }] +} +``` + +### 2. Export Required Classes + +```typescript +// Export the proxy for tool execution (required for codemode) +export { CodeModeProxy } from "@cloudflare/codemode/ai"; + +// Define outbound fetch handler for security filtering +export const globalOutbound = { + fetch: async (input: string | URL | RequestInfo, init?: RequestInit) => { + const url = new URL( + typeof input === "string" + ? input + : typeof input === "object" && "url" in input + ? input.url + : input.toString() + ); + // Block certain domains if needed + if (url.hostname === "blocked.example.com") { + return new Response("Not allowed", { status: 403 }); + } + return fetch(input, init); + } +}; +``` + +### 3. Install Dependencies + +```bash +npm install @cloudflare/codemode ai @ai-sdk/openai zod +``` + +### 4. Use Code Mode in Agent + +```typescript +import { Agent } from "agents"; +import { experimental_codemode as codemode } from "@cloudflare/codemode/ai"; +import { streamText, tool, convertToModelMessages } from "ai"; +import { openai } from "@ai-sdk/openai"; +import { env } from "cloudflare:workers"; +import { z } from "zod"; + +const tools = { + getWeather: tool({ + description: "Get weather for a location", + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather: ${location} 72°F` + }), + sendEmail: tool({ + description: "Send an email", + parameters: z.object({ to: z.string(), subject: z.string(), body: z.string() }), + execute: async ({ to, subject, body }) => `Email sent to ${to}` + }) +}; + +export class MyAgent extends Agent { + tools = {}; + + // Method called by codemode proxy + callTool(functionName: string, args: unknown[]) { + return this.tools[functionName]?.execute?.(args, { + abortSignal: new AbortController().signal, + toolCallId: "codemode", + messages: [] + }); + } + + async onChatMessage() { + this.tools = { ...tools, ...this.mcp.getAITools() }; + + const { prompt, tools: wrappedTools } = await codemode({ + prompt: "You are a helpful assistant...", + tools: this.tools, + globalOutbound: env.globalOutbound, + loader: env.LOADER, + proxy: this.ctx.exports.CodeModeProxy({ + props: { + binding: "MyAgent", // Class name + name: this.name, // Instance name + callback: "callTool" // Method to call + } + }) + }); + + const result = streamText({ + system: prompt, + model: openai("gpt-4o"), + messages: await convertToModelMessages(this.state.messages), + tools: wrappedTools // Use wrapped tools, not original + }); + + // ... handle stream + } +} +``` + +## Generated Code Example + +When user asks "Check the weather in NYC and email me the forecast", codemode generates: + +```javascript +async function executeTask() { + const weather = await codemode.getWeather({ location: "NYC" }); + + await codemode.sendEmail({ + to: "user@example.com", + subject: "NYC Weather Forecast", + body: `Current weather: ${weather}` + }); + + return { success: true, weather }; +} +``` + +## MCP Server Orchestration + +Code Mode excels at orchestrating multiple MCP servers: + +```javascript +async function executeTask() { + // Query file system MCP + const files = await codemode.listFiles({ path: "/projects" }); + + // Query database MCP + const status = await codemode.queryDatabase({ + query: "SELECT * FROM projects WHERE name = ?", + params: [files[0].name] + }); + + // Conditional logic based on results + if (status.length === 0) { + await codemode.createTask({ + title: `Review: ${files[0].name}`, + priority: "high" + }); + } + + return { files, status }; +} +``` + +## When to Use + +| Scenario | Use Code Mode? | +|----------|---------------| +| Single tool call | No | +| Chained tool calls | Yes | +| Conditional logic across tools | Yes | +| MCP multi-server workflows | Yes | +| Token budget constrained | Yes | +| Simple Q&A chat | No | + +## Limitations + +- Experimental - API may change +- Requires Cloudflare Workers +- JavaScript execution only (Python planned) +- Requires additional wrangler config diff --git a/plugins/cloudflare/skills/agents-sdk/references/email.md b/plugins/cloudflare/skills/agents-sdk/references/email.md new file mode 100644 index 0000000..0cf2b83 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/email.md @@ -0,0 +1,146 @@ +# Email Handling + +Fetch `docs/email.md` from `https://github.com/cloudflare/agents/tree/main/docs` for complete documentation. + +## Overview + +Agents receive and reply to emails via Cloudflare Email Routing. + +## Wrangler Configuration + +```jsonc +{ + "durable_objects": { + "bindings": [{ "name": "EmailAgent", "class_name": "EmailAgent" }] + }, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["EmailAgent"] }], + "send_email": [ + { "name": "SEB", "destination_address": "reply@yourdomain.com" } + ] +} +``` + +## Basic Email Handler + +```typescript +import { Agent } from "agents"; +import { type AgentEmail } from "agents/email"; +import PostalMime from "postal-mime"; + +export class EmailAgent extends Agent { + async onEmail(email: AgentEmail) { + const raw = await email.getRaw(); + const parsed = await PostalMime.parse(raw); + + console.log("From:", email.from); + console.log("Subject:", parsed.subject); + + await this.replyToEmail(email, { + fromName: "My Agent", + subject: `Re: ${parsed.subject}`, + body: "Thanks for your email!" + }); + } +} +``` + +## Routing Emails + +```typescript +import { routeAgentRequest, routeAgentEmail } from "agents"; +import { createAddressBasedEmailResolver } from "agents/email"; + +export default { + async email(message, env) { + await routeAgentEmail(message, env, { + resolver: createAddressBasedEmailResolver("EmailAgent") + }); + }, + + async fetch(request, env) { + return routeAgentRequest(request, env) ?? new Response("Not found", { status: 404 }); + } +}; +``` + +## Resolvers + +### Address-Based (Inbound Mail) + +Routes based on recipient address: + +```typescript +import { createAddressBasedEmailResolver } from "agents/email"; + +const resolver = createAddressBasedEmailResolver("EmailAgent"); +// support@example.com → EmailAgent, instance "support" +// NotificationAgent+user123@example.com → NotificationAgent, instance "user123" +``` + +### Secure Reply (Reply Flows) + +Verifies replies are authentic using HMAC-SHA256 signatures: + +```typescript +import { createSecureReplyEmailResolver } from "agents/email"; + +const resolver = createSecureReplyEmailResolver(env.EMAIL_SECRET, { + maxAge: 7 * 24 * 60 * 60, // 7 days (default: 30 days) + onInvalidSignature: (email, reason) => { + console.warn(`Invalid signature from ${email.from}: ${reason}`); + } +}); +``` + +Sign outbound emails to enable secure reply routing: + +```typescript +await this.replyToEmail(email, { + fromName: "My Agent", + body: "Thanks!", + secret: this.env.EMAIL_SECRET // Signs headers for secure reply routing +}); +``` + +### Catch-All (Single Instance) + +Routes all emails to one agent instance: + +```typescript +import { createCatchAllEmailResolver } from "agents/email"; + +const resolver = createCatchAllEmailResolver("EmailAgent", "default"); +``` + +### Combining Resolvers + +```typescript +async email(message, env) { + const secureReply = createSecureReplyEmailResolver(env.EMAIL_SECRET); + const addressBased = createAddressBasedEmailResolver("EmailAgent"); + + await routeAgentEmail(message, env, { + resolver: async (email, env) => { + // Try secure reply first + const result = await secureReply(email, env); + if (result) return result; + // Fall back to address-based + return addressBased(email, env); + } + }); +} +``` + +## Utilities + +```typescript +import { isAutoReplyEmail } from "agents/email"; + +async onEmail(email: AgentEmail) { + if (isAutoReplyEmail(email.headers)) { + // Skip auto-replies (vacation, out-of-office, etc.) + return; + } + // Process email... +} +``` diff --git a/plugins/cloudflare/skills/agents-sdk/references/mcp.md b/plugins/cloudflare/skills/agents-sdk/references/mcp.md new file mode 100644 index 0000000..9fed33c --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/mcp.md @@ -0,0 +1,154 @@ +# MCP Server Integration + +Fetch `docs/mcp-client.md` and `docs/mcp-servers.md` from `https://github.com/cloudflare/agents/tree/main/docs` for complete documentation. + +Agents include a multi-server MCP client for connecting to external MCP servers. + +## Add an MCP Server + +```typescript +import { Agent, callable } from "agents"; + +export class MyAgent extends Agent { + @callable() + async addServer(name: string, url: string) { + // Options-based API (recommended) + const result = await this.addMcpServer(name, url, { + callbackHost: "https://my-worker.workers.dev", + transport: { headers: { Authorization: "Bearer ..." } } + }); + + if (result.state === "authenticating") { + // OAuth required - redirect user to result.authUrl + return { needsAuth: true, authUrl: result.authUrl }; + } + + return { ready: true, id: result.id }; + } +} +``` + +## Use MCP Tools + +```typescript +async onChatMessage() { + // Get AI-compatible tools from all connected MCP servers + const mcpTools = this.mcp.getAITools(); + + const allTools = { + ...localTools, + ...mcpTools + }; + + const result = streamText({ + model: openai("gpt-4o"), + messages: await convertToModelMessages(this.messages), + tools: allTools + }); + + return result.toUIMessageStreamResponse(); +} +``` + +## List MCP Resources + +```typescript +// List all registered servers +const servers = this.mcp.listServers(); + +// List tools from all servers +const tools = this.mcp.listTools(); + +// List resources +const resources = this.mcp.listResources(); + +// List prompts +const prompts = this.mcp.listPrompts(); +``` + +## Remove Server + +```typescript +await this.removeMcpServer(serverId); +``` + +## Building an MCP Server + +Use `McpAgent` from the SDK to create an MCP server. + +**Install dependencies:** +```bash +npm install @modelcontextprotocol/sdk zod +``` + +**Wrangler config:** +```jsonc +{ + "durable_objects": { + "bindings": [{ "name": "MyMCP", "class_name": "MyMCP" }] + }, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["MyMCP"] }] +} +``` + +**Server implementation:** +```typescript +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { McpAgent } from "agents/mcp"; +import { z } from "zod"; + +type State = { counter: number }; + +export class MyMCP extends McpAgent { + server = new McpServer({ + name: "MyMCPServer", + version: "1.0.0" + }); + + initialState = { counter: 0 }; + + async init() { + // Register a resource + this.server.resource("counter", "mcp://resource/counter", (uri) => ({ + contents: [{ text: String(this.state.counter), uri: uri.href }] + })); + + // Register a tool + this.server.registerTool( + "increment", + { + description: "Increment the counter", + inputSchema: { amount: z.number().default(1) } + }, + async ({ amount }) => { + this.setState({ counter: this.state.counter + amount }); + return { + content: [{ text: `Counter: ${this.state.counter}`, type: "text" }] + }; + } + ); + } +} +``` + +## Serve MCP Server + +```typescript +export default { + fetch(request: Request, env: Env, ctx: ExecutionContext) { + const url = new URL(request.url); + + // SSE transport (legacy) + if (url.pathname.startsWith("/sse")) { + return MyMCP.serveSSE("/sse", { binding: "MyMCP" }).fetch(request, env, ctx); + } + + // Streamable HTTP transport (recommended) + if (url.pathname.startsWith("/mcp")) { + return MyMCP.serve("/mcp", { binding: "MyMCP" }).fetch(request, env, ctx); + } + + return new Response("Not found", { status: 404 }); + } +}; +``` diff --git a/plugins/cloudflare/skills/agents-sdk/references/state-scheduling.md b/plugins/cloudflare/skills/agents-sdk/references/state-scheduling.md new file mode 100644 index 0000000..97f9581 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/state-scheduling.md @@ -0,0 +1,164 @@ +# State & Scheduling + +Fetch `docs/state.md` and `docs/scheduling.md` from `https://github.com/cloudflare/agents/tree/main/docs` for complete documentation. + +## State Management + +State persists to SQLite and broadcasts to connected clients automatically. + +### Define Typed State + +```typescript +type State = { + count: number; + items: string[]; +}; + +export class MyAgent extends Agent { + initialState: State = { count: 0, items: [] }; +} +``` + +### Read and Update + +```typescript +// Read (lazy-loaded from SQLite) +const count = this.state.count; + +// Write (sync, persists, broadcasts) +this.setState({ count: this.state.count + 1 }); +``` + +### Validation Hook + +`validateStateChange()` runs synchronously before state persists. Throw to reject the update. + +```typescript +validateStateChange(nextState: State, source: Connection | "server") { + if (nextState.count < 0) { + throw new Error("Count cannot be negative"); + } +} +``` + +### Execution Order + +1. `validateStateChange(nextState, source)` - sync, gating +2. State persisted to SQLite +3. State broadcast to connected clients +4. `onStateUpdate(nextState, source)` - async via `ctx.waitUntil`, non-gating + +### Client-Side Sync (React) + +```tsx +import { useAgent } from "agents/react"; + +function App() { + const [state, setLocalState] = useState({ count: 0 }); + + const agent = useAgent({ + agent: "MyAgent", + name: "instance-1", + onStateUpdate: (newState) => setLocalState(newState) + }); + + return ; +} +``` + +## SQL API + +Direct SQLite access for custom queries: + +```typescript +// Create table +this.sql` + CREATE TABLE IF NOT EXISTS items ( + id TEXT PRIMARY KEY, + name TEXT, + created_at INTEGER DEFAULT (unixepoch()) + ) +`; + +// Insert +this.sql`INSERT INTO items (id, name) VALUES (${id}, ${name})`; + +// Query with types +const items = this.sql<{ id: string; name: string }>` + SELECT * FROM items WHERE name LIKE ${`%${search}%`} +`; +``` + +## Scheduling + +### Schedule Types + +| Mode | Syntax | Use Case | +|------|--------|----------| +| Delay | `this.schedule(60, ...)` | Run in 60 seconds | +| Date | `this.schedule(new Date(...), ...)` | Run at specific time | +| Cron | `this.schedule("0 8 * * *", ...)` | Recurring schedule | +| Interval | `this.scheduleEvery(30, ...)` | Fixed interval (every 30s) | + +### Examples + +```typescript +// Delay (seconds) +await this.schedule(60, "checkStatus", { id: "abc123" }); + +// Specific date +await this.schedule(new Date("2025-12-25T00:00:00Z"), "sendGreeting", { to: "user" }); + +// Cron (recurring) +await this.schedule("0 9 * * 1-5", "weekdayReport", {}); + +// Fixed interval (every 30 seconds, overlap prevention built-in) +await this.scheduleEvery(30, "pollUpdates"); +await this.scheduleEvery(300, "syncData", { source: "api" }); +``` + +### Handler + +```typescript +async sendGreeting(payload: { to: string }, schedule: Schedule) { + console.log(`Sending greeting to ${payload.to}`); + // Cron schedules auto-reschedule; one-time schedules are deleted +} +``` + +### Manage Schedules + +```typescript +const schedules = this.getSchedules(); +const crons = this.getSchedules({ type: "cron" }); +await this.cancelSchedule(schedule.id); +``` + +## Lifecycle Callbacks + +```typescript +export class MyAgent extends Agent { + async onStart() { + // Agent started or woke from hibernation + } + + onConnect(conn: Connection, ctx: ConnectionContext) { + // WebSocket connected + } + + onMessage(conn: Connection, message: WSMessage) { + // WebSocket message (non-RPC) + } + + onStateUpdate(state: State, source: Connection | "server") { + // State changed (async, non-blocking) + } + + onError(error: unknown) { + // Error handler + throw error; // Re-throw to propagate + } +} +``` diff --git a/plugins/cloudflare/skills/agents-sdk/references/streaming-chat.md b/plugins/cloudflare/skills/agents-sdk/references/streaming-chat.md new file mode 100644 index 0000000..249e927 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/streaming-chat.md @@ -0,0 +1,178 @@ +# Streaming Chat with AIChatAgent + +Fetch `docs/resumable-streaming.md` and `docs/client-sdk.md` from `https://github.com/cloudflare/agents/tree/main/docs` for complete documentation. + +`AIChatAgent` provides streaming chat with automatic message persistence and resumable streams. + +## Basic Chat Agent + +```typescript +import { AIChatAgent } from "@cloudflare/ai-chat"; +import { streamText, convertToModelMessages } from "ai"; +import { openai } from "@ai-sdk/openai"; + +export class Chat extends AIChatAgent { + async onChatMessage(onFinish) { + const result = streamText({ + model: openai("gpt-4o"), + messages: await convertToModelMessages(this.messages), + onFinish + }); + return result.toUIMessageStreamResponse(); + } +} +``` + +## With Custom System Prompt + +```typescript +export class Chat extends AIChatAgent { + async onChatMessage(onFinish) { + const result = streamText({ + model: openai("gpt-4o"), + system: "You are a helpful assistant specializing in...", + messages: await convertToModelMessages(this.messages), + onFinish + }); + return result.toUIMessageStreamResponse(); + } +} +``` + +## With Tools + +```typescript +import { tool } from "ai"; +import { z } from "zod"; + +const tools = { + getWeather: tool({ + description: "Get weather for a location", + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: 72°F, sunny` + }) +}; + +export class Chat extends AIChatAgent { + async onChatMessage(onFinish) { + const result = streamText({ + model: openai("gpt-4o"), + messages: await convertToModelMessages(this.messages), + tools, + onFinish + }); + return result.toUIMessageStreamResponse(); + } +} +``` + +## Custom UI Message Stream + +For more control, use `createUIMessageStream`: + +```typescript +import { createUIMessageStream, createUIMessageStreamResponse } from "ai"; + +export class Chat extends AIChatAgent { + async onChatMessage(onFinish) { + const stream = createUIMessageStream({ + execute: async ({ writer }) => { + const result = streamText({ + model: openai("gpt-4o"), + messages: await convertToModelMessages(this.messages), + onFinish + }); + writer.merge(result.toUIMessageStream()); + } + }); + return createUIMessageStreamResponse({ stream }); + } +} +``` + +## Resumable Streaming + +Streams automatically resume if client disconnects and reconnects: + +1. Chunks buffered to SQLite during streaming +2. On reconnect, buffered chunks sent immediately +3. Live streaming continues from where it left off + +**Enabled by default.** To disable: + +```tsx +const { messages } = useAgentChat({ agent, resume: false }); +``` + +## React Client + +```tsx +import { useAgent } from "agents/react"; +import { useAgentChat } from "@cloudflare/ai-chat/react"; + +function ChatUI() { + const agent = useAgent({ + agent: "Chat", + name: "my-chat-session" + }); + + const { + messages, + input, + handleInputChange, + handleSubmit, + status + } = useAgentChat({ agent }); + + return ( +
+ {messages.map((m) => ( +
+ {m.role}: {m.content} +
+ ))} + +
+ + +
+
+ ); +} +``` + +## Streaming RPC Methods + +For non-chat streaming, use `@callable({ streaming: true })`: + +```typescript +import { Agent, callable, StreamingResponse } from "agents"; + +export class MyAgent extends Agent { + @callable({ streaming: true }) + async streamData(stream: StreamingResponse, query: string) { + for (let i = 0; i < 10; i++) { + stream.send(`Result ${i}: ${query}`); + await sleep(100); + } + stream.close(); + } +} +``` + +Client receives streamed messages via WebSocket RPC. + +## Status Values + +`useAgentChat` status: + +| Status | Meaning | +|--------|---------| +| `ready` | Idle, ready for input | +| `streaming` | Response streaming | +| `submitted` | Request sent, waiting | +| `error` | Error occurred | diff --git a/plugins/cloudflare/skills/agents-sdk/references/workflows.md b/plugins/cloudflare/skills/agents-sdk/references/workflows.md new file mode 100644 index 0000000..791cfa9 --- /dev/null +++ b/plugins/cloudflare/skills/agents-sdk/references/workflows.md @@ -0,0 +1,132 @@ +# Workflows Integration + +Fetch `docs/workflows.md` from `https://github.com/cloudflare/agents/tree/main/docs` for complete documentation. + +## Overview + +Agents handle real-time communication; Workflows handle durable execution. Together they enable: + +- Long-running background tasks with automatic retries +- Human-in-the-loop approval flows +- Multi-step pipelines that survive failures + +| Use Case | Recommendation | +|----------|----------------| +| Chat/messaging | Agent only | +| Quick API calls (<30s) | Agent only | +| Background processing (<30s) | Agent `queue()` | +| Long-running tasks (>30s) | Agent + Workflow | +| Human approval flows | Agent + Workflow | + +## AgentWorkflow Base Class + +```typescript +import { AgentWorkflow } from "agents/workflows"; +import type { AgentWorkflowEvent, AgentWorkflowStep } from "agents/workflows"; + +type TaskParams = { taskId: string; data: string }; + +export class ProcessingWorkflow extends AgentWorkflow { + async run(event: AgentWorkflowEvent, step: AgentWorkflowStep) { + const params = event.payload; + + // Durable step - retries on failure + const result = await step.do("process", async () => { + return processData(params.data); + }); + + // Non-durable: progress reporting + await this.reportProgress({ step: "process", percent: 0.5 }); + + // Non-durable: broadcast to connected clients + this.broadcastToClients({ type: "update", taskId: params.taskId }); + + // Durable: merge state via step + await step.mergeAgentState({ lastProcessed: params.taskId }); + + // Durable: report completion + await step.reportComplete(result); + + return result; + } +} +``` + +## Wrangler Configuration + +```jsonc +{ + "workflows": [ + { "name": "processing-workflow", "binding": "PROCESSING_WORKFLOW", "class_name": "ProcessingWorkflow" } + ], + "durable_objects": { + "bindings": [{ "name": "MyAgent", "class_name": "MyAgent" }] + }, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["MyAgent"] }] +} +``` + +## Agent Methods for Workflows + +```typescript +// Start a workflow +const instance = await this.runWorkflow("ProcessingWorkflow", { taskId: "123", data: "..." }); + +// Send event to waiting workflow +await this.sendWorkflowEvent("ProcessingWorkflow", workflowId, { type: "approve" }); + +// Query workflows +const workflow = await this.getWorkflow(workflowId); +const workflows = await this.getWorkflows({ status: "running" }); + +// Control workflows +await this.approveWorkflow(workflowId); +await this.rejectWorkflow(workflowId); +await this.terminateWorkflow(workflowId); +await this.pauseWorkflow(workflowId); +await this.resumeWorkflow(workflowId); + +// Delete workflows +await this.deleteWorkflow(workflowId); +await this.deleteWorkflows({ status: "complete", before: new Date(...) }); +``` + +## Lifecycle Callbacks + +```typescript +export class MyAgent extends Agent { + async onWorkflowProgress(workflowName: string, workflowId: string, progress: unknown) { + // Workflow reported progress via this.reportProgress() + this.broadcast({ type: "progress", workflowId, progress }); + } + + async onWorkflowComplete(workflowName: string, workflowId: string, result?: unknown) { + // Workflow finished successfully + } + + async onWorkflowError(workflowName: string, workflowId: string, error: Error) { + // Workflow failed + } + + async onWorkflowEvent(workflowName: string, workflowId: string, event: unknown) { + // Workflow received an event via sendWorkflowEvent() + } +} +``` + +## Human-in-the-Loop + +```typescript +// In workflow: wait for approval +const approved = await step.waitForEvent<{ approved: boolean }>("approval", { + timeout: "7d" +}); + +if (!approved.approved) { + throw new Error("Rejected"); +} + +// From agent: approve or reject +await this.approveWorkflow(workflowId); // Sends { approved: true } +await this.rejectWorkflow(workflowId); // Sends { approved: false } +``` diff --git a/plugins/cloudflare/skills/cloudflare/LICENSE.txt b/plugins/cloudflare/skills/cloudflare/LICENSE.txt new file mode 100644 index 0000000..13e25df --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of + any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don\'t include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/plugins/cloudflare/skills/cloudflare/SKILL.md b/plugins/cloudflare/skills/cloudflare/SKILL.md new file mode 100644 index 0000000..870f9a2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/SKILL.md @@ -0,0 +1,228 @@ +--- +name: cloudflare +description: Comprehensive Cloudflare platform skill covering Workers, Pages, storage (KV, D1, R2), AI (Workers AI, Vectorize, Agents SDK), networking (Tunnel, Spectrum), security (WAF, DDoS), and infrastructure-as-code (Terraform, Pulumi). Use for any Cloudflare development task. Biases towards retrieval from Cloudflare docs over pre-trained knowledge. +references: + - workers + - pages + - d1 + - durable-objects + - workers-ai +--- + +# Cloudflare Platform Skill + +Consolidated skill for building on the Cloudflare platform. Use decision trees below to find the right product, then load detailed references. + +Your knowledge of Cloudflare APIs, types, limits, and pricing may be outdated. **Prefer retrieval over pre-training** — the references in this skill are starting points, not source of truth. + +## Retrieval Sources + +Fetch the **latest** information before citing specific numbers, API signatures, or configuration options. Do not rely on baked-in knowledge or these reference files alone. + +| Source | How to retrieve | Use for | +|--------|----------------|---------| +| Cloudflare docs | `cloudflare-docs` search tool or `https://developers.cloudflare.com/` | Limits, pricing, API reference, compatibility dates/flags | +| Workers types | `npm pack @cloudflare/workers-types` or check `node_modules` | Type signatures, binding shapes, handler types | +| Wrangler config schema | `node_modules/wrangler/config-schema.json` | Config fields, binding shapes, allowed values | +| Product changelogs | `https://developers.cloudflare.com/changelog/` | Recent changes to limits, features, deprecations | + +When a reference file and the docs disagree, **trust the docs**. This is especially important for: numeric limits, pricing tiers, type signatures, and configuration options. + +## Quick Decision Trees + +### "I need to run code" + +``` +Need to run code? +├─ Serverless functions at the edge → workers/ +├─ Full-stack web app with Git deploys → pages/ +├─ Stateful coordination/real-time → durable-objects/ +├─ Long-running multi-step jobs → workflows/ +├─ Run containers → containers/ +├─ Multi-tenant (customers deploy code) → workers-for-platforms/ +├─ Scheduled tasks (cron) → cron-triggers/ +├─ Lightweight edge logic (modify HTTP) → snippets/ +├─ Process Worker execution events (logs/observability) → tail-workers/ +└─ Optimize latency to backend infrastructure → smart-placement/ +``` + +### "I need to store data" + +``` +Need storage? +├─ Key-value (config, sessions, cache) → kv/ +├─ Relational SQL → d1/ (SQLite) or hyperdrive/ (existing Postgres/MySQL) +├─ Object/file storage (S3-compatible) → r2/ +├─ Message queue (async processing) → queues/ +├─ Vector embeddings (AI/semantic search) → vectorize/ +├─ Strongly-consistent per-entity state → durable-objects/ (DO storage) +├─ Secrets management → secrets-store/ +├─ Streaming ETL to R2 → pipelines/ +└─ Persistent cache (long-term retention) → cache-reserve/ +``` + +### "I need AI/ML" + +``` +Need AI? +├─ Run inference (LLMs, embeddings, images) → workers-ai/ +├─ Vector database for RAG/search → vectorize/ +├─ Build stateful AI agents → agents-sdk/ +├─ Gateway for any AI provider (caching, routing) → ai-gateway/ +└─ AI-powered search widget → ai-search/ +``` + +### "I need networking/connectivity" + +``` +Need networking? +├─ Expose local service to internet → tunnel/ +├─ TCP/UDP proxy (non-HTTP) → spectrum/ +├─ WebRTC TURN server → turn/ +├─ Private network connectivity → network-interconnect/ +├─ Optimize routing → argo-smart-routing/ +├─ Optimize latency to backend (not user) → smart-placement/ +└─ Real-time video/audio → realtimekit/ or realtime-sfu/ +``` + +### "I need security" + +``` +Need security? +├─ Web Application Firewall → waf/ +├─ DDoS protection → ddos/ +├─ Bot detection/management → bot-management/ +├─ API protection → api-shield/ +├─ CAPTCHA alternative → turnstile/ +└─ Credential leak detection → waf/ (managed ruleset) +``` + +### "I need media/content" + +``` +Need media? +├─ Image optimization/transformation → images/ +├─ Video streaming/encoding → stream/ +├─ Browser automation/screenshots → browser-rendering/ +└─ Third-party script management → zaraz/ +``` + +### "I need analytics/metrics data" + +``` +Need analytics? +├─ Query across all Cloudflare products (HTTP, Workers, DNS, etc.) → graphql-api/ +├─ Custom high-cardinality metrics from Workers → analytics-engine/ +├─ Client-side (RUM) performance data → web-analytics/ +├─ Workers Logs and real-time debugging → observability/ +└─ Raw logs (Logpush to external tools) → Cloudflare docs +``` + +### "I need infrastructure-as-code" + +``` +Need IaC? → pulumi/ (Pulumi), terraform/ (Terraform), or api/ (REST API) +``` + +## Product Index + +### Compute & Runtime +| Product | Reference | +|---------|-----------| +| Workers | `references/workers/` | +| Pages | `references/pages/` | +| Pages Functions | `references/pages-functions/` | +| Durable Objects | `references/durable-objects/` | +| Workflows | `references/workflows/` | +| Containers | `references/containers/` | +| Workers for Platforms | `references/workers-for-platforms/` | +| Cron Triggers | `references/cron-triggers/` | +| Tail Workers | `references/tail-workers/` | +| Snippets | `references/snippets/` | +| Smart Placement | `references/smart-placement/` | + +### Storage & Data +| Product | Reference | +|---------|-----------| +| KV | `references/kv/` | +| D1 | `references/d1/` | +| R2 | `references/r2/` | +| Queues | `references/queues/` | +| Hyperdrive | `references/hyperdrive/` | +| DO Storage | `references/do-storage/` | +| Secrets Store | `references/secrets-store/` | +| Pipelines | `references/pipelines/` | +| R2 Data Catalog | `references/r2-data-catalog/` | +| R2 SQL | `references/r2-sql/` | + +### AI & Machine Learning +| Product | Reference | +|---------|-----------| +| Workers AI | `references/workers-ai/` | +| Vectorize | `references/vectorize/` | +| Agents SDK | `references/agents-sdk/` | +| AI Gateway | `references/ai-gateway/` | +| AI Search | `references/ai-search/` | + +### Networking & Connectivity +| Product | Reference | +|---------|-----------| +| Tunnel | `references/tunnel/` | +| Spectrum | `references/spectrum/` | +| TURN | `references/turn/` | +| Network Interconnect | `references/network-interconnect/` | +| Argo Smart Routing | `references/argo-smart-routing/` | +| Workers VPC | `references/workers-vpc/` | + +### Security +| Product | Reference | +|---------|-----------| +| WAF | `references/waf/` | +| DDoS Protection | `references/ddos/` | +| Bot Management | `references/bot-management/` | +| API Shield | `references/api-shield/` | +| Turnstile | `references/turnstile/` | + +### Media & Content +| Product | Reference | +|---------|-----------| +| Images | `references/images/` | +| Stream | `references/stream/` | +| Browser Rendering | `references/browser-rendering/` | +| Zaraz | `references/zaraz/` | + +### Real-Time Communication +| Product | Reference | +|---------|-----------| +| RealtimeKit | `references/realtimekit/` | +| Realtime SFU | `references/realtime-sfu/` | + +### Developer Tools +| Product | Reference | +|---------|-----------| +| Wrangler | `references/wrangler/` | +| Miniflare | `references/miniflare/` | +| C3 | `references/c3/` | +| Observability | `references/observability/` | +| GraphQL Analytics API | `references/graphql-api/` | +| Analytics Engine | `references/analytics-engine/` | +| Web Analytics | `references/web-analytics/` | +| Sandbox | `references/sandbox/` | +| Workerd | `references/workerd/` | +| Workers Playground | `references/workers-playground/` | + +### Infrastructure as Code +| Product | Reference | +|---------|-----------| +| Pulumi | `references/pulumi/` | +| Terraform | `references/terraform/` | +| API | `references/api/` | + +### Other Services +| Product | Reference | +|---------|-----------| +| Email Routing | `references/email-routing/` | +| Email Workers | `references/email-workers/` | +| Static Assets | `references/static-assets/` | +| Bindings | `references/bindings/` | +| Cache Reserve | `references/cache-reserve/` | diff --git a/plugins/cloudflare/skills/cloudflare/agents/openai.yaml b/plugins/cloudflare/skills/cloudflare/agents/openai.yaml new file mode 100644 index 0000000..43075bb --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Cloudflare" + short_description: "Choose the right Cloudflare product and workflow" + icon_small: "./assets/cloudflare-small.svg" + icon_large: "./assets/cloudflare.png" + default_prompt: "Use Cloudflare context to choose the right product, verify setup, or plan the right deployment, data, or analytics workflow without guessing infrastructure." diff --git a/plugins/cloudflare/skills/cloudflare/assets/cloudflare-small.svg b/plugins/cloudflare/skills/cloudflare/assets/cloudflare-small.svg new file mode 100644 index 0000000..7feedbc --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/assets/cloudflare-small.svg @@ -0,0 +1,3 @@ + diff --git a/plugins/cloudflare/skills/cloudflare/assets/cloudflare.png b/plugins/cloudflare/skills/cloudflare/assets/cloudflare.png new file mode 100644 index 0000000..cf23088 Binary files /dev/null and b/plugins/cloudflare/skills/cloudflare/assets/cloudflare.png differ diff --git a/plugins/cloudflare/skills/cloudflare/references/agents-sdk/README.md b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/README.md new file mode 100644 index 0000000..c30d098 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/README.md @@ -0,0 +1,89 @@ +# Cloudflare Agents SDK + +Cloudflare Agents SDK enables building AI-powered agents on Durable Objects with state, WebSockets, SQL, scheduling, and AI integration. + +## Core Value +Build stateful, globally distributed AI agents with persistent memory, real-time connections, scheduled tasks, and async workflows. + +## When to Use +- Persistent state + memory required +- Real-time WebSocket connections +- Long-running workflows (minutes/hours) +- Chat interfaces with AI models +- Scheduled/recurring tasks with state +- DB queries with agent state + +## What Type of Agent? + +| Use Case | Class | Key Features | +|----------|-------|--------------| +| AI chat interface | `AIChatAgent` | Auto-streaming, tools, message history, resumable | +| MCP tool provider | `Agent` + MCP | Expose tools to AI systems | +| Custom logic/routing | `Agent` | Full control, WebSockets, email, SQL | +| Real-time collaboration | `Agent` | WebSocket state, broadcasts | +| Email processing | `Agent` | `onEmail()` handler | + +## Quick Start + +**AI Chat Agent:** +```typescript +import { AIChatAgent } from "agents"; +import { openai } from "@ai-sdk/openai"; + +export class ChatAgent extends AIChatAgent { + async onChatMessage(onFinish) { + return this.streamText({ + model: openai("gpt-4"), + messages: this.messages, + onFinish, + }); + } +} +``` + +**Base Agent:** +```typescript +import { Agent } from "agents"; + +export class MyAgent extends Agent { + onStart() { + this.sql`CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY)`; + } + + async onRequest(request: Request) { + return Response.json({ state: this.state }); + } +} +``` + +## Reading Order + +| Task | Files to Read | +|------|---------------| +| Quick start | README only | +| Build chat agent | README → api.md (AIChatAgent) → patterns.md | +| Setup project | README → configuration.md | +| Add React frontend | README → api.md (Client Hooks) → patterns.md | +| Build MCP server | api.md (MCP) → patterns.md | +| Background tasks | api.md (Scheduling, Task Queue) → patterns.md | +| Debug issues | gotchas.md | + +## Package Entry Points + +| Import | Purpose | +|--------|---------| +| `agents` | Server-side Agent classes, lifecycle | +| `agents/react` | `useAgent()` hook for WebSocket connections | +| `agents/ai-react` | `useAgentChat()` hook for AI chat UIs | + +## In This Reference +- [configuration.md](./configuration.md) - SDK setup, wrangler config, routing +- [api.md](./api.md) - Agent classes, lifecycle, client hooks +- [patterns.md](./patterns.md) - Common workflows, best practices +- [gotchas.md](./gotchas.md) - Common issues, limits + +## See Also +- durable-objects - Agent infrastructure +- d1 - External database integration +- workers-ai - AI model integration +- vectorize - Vector search for RAG patterns \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/agents-sdk/api.md b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/api.md new file mode 100644 index 0000000..9e16aa6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/api.md @@ -0,0 +1,190 @@ +# API Reference + +## Agent Classes + +### AIChatAgent + +For AI chat with auto-streaming, message history, tools, resumable streaming. + +```ts +import { AIChatAgent } from "agents"; +import { openai } from "@ai-sdk/openai"; + +export class ChatAgent extends AIChatAgent { + async onChatMessage(onFinish) { + return this.streamText({ + model: openai("gpt-4"), + messages: this.messages, // Auto-managed message history + tools: { + getWeather: { + description: "Get weather", + parameters: z.object({ city: z.string() }), + execute: async ({ city }) => `Sunny, 72°F in ${city}` + } + }, + onFinish, // Persist response to this.messages + }); + } +} +``` + +### Agent (Base Class) + +Full control for custom logic, WebSockets, email, and SQL. + +```ts +import { Agent } from "agents"; + +export class MyAgent extends Agent { + // Lifecycle methods below +} +``` + +**Type params:** `Agent` - Env bindings, agent state, connection state + +## Lifecycle Hooks + +```ts +onStart() { // Init/restart + this.sql`CREATE TABLE IF NOT EXISTS users (id TEXT, name TEXT)`; +} + +async onRequest(req: Request) { // HTTP + const {pathname} = new URL(req.url); + if (pathname === "/users") return Response.json(this.sql<{id,name}>`SELECT * FROM users`); + return new Response("Not found", {status: 404}); +} + +async onConnect(conn: Connection, ctx: ConnectionContext) { // WebSocket + conn.accept(); + conn.setState({userId: ctx.request.headers.get("X-User-ID")}); + conn.send(JSON.stringify({type: "connected", state: this.state})); +} + +async onMessage(conn: Connection, msg: WSMessage) { // WS messages + const m = JSON.parse(msg as string); + this.setState({messages: [...this.state.messages, m]}); + this.connections.forEach(c => c.send(JSON.stringify(m))); +} + +async onEmail(email: AgentEmail) { // Email routing + this.sql`INSERT INTO emails (from_addr,subject,body) VALUES (${email.from},${email.headers.get("subject")},${await email.text()})`; +} +``` + +## State, SQL, Scheduling + +```ts +// State +this.setState({count: 42}); // Auto-syncs +this.setState({...this.state, count: this.state.count + 1}); + +// SQL (parameterized queries prevent injection) +this.sql`CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY, name TEXT)`; +this.sql`INSERT INTO users (id,name) VALUES (${userId},${name})`; +const users = this.sql<{id,name}>`SELECT * FROM users WHERE id = ${userId}`; + +// Scheduling +await this.schedule(new Date("2026-12-25"), "sendGreeting", {msg:"Hi"}); // Date +await this.schedule(60, "checkStatus", {}); // Delay (sec) +await this.schedule("0 0 * * *", "dailyCleanup", {}); // Cron +await this.cancelSchedule(scheduleId); +``` + +## RPC Methods (@callable) + +```ts +import { Agent, callable } from "agents"; + +export class MyAgent extends Agent { + @callable() + async processTask(input: {text: string}): Promise<{result: string}> { + return { result: await this.env.AI.run("@cf/meta/llama-3.1-8b-instruct", {prompt: input.text}) }; + } +} +// Client: const result = await agent.processTask({ text: "Hello" }); +// Must return JSON-serializable values +``` + +## Connections & AI + +```ts +// Connections (type: Agent) +this.connections.forEach(c => c.send(JSON.stringify(msg))); // Broadcast +conn.setState({userId:"123"}); conn.close(1000, "Goodbye"); + +// Workers AI +const r = await this.env.AI.run("@cf/meta/llama-3.1-8b-instruct", {prompt}); + +// Manual streaming (prefer AIChatAgent) +const stream = await client.chat.completions.create({model: "gpt-4", messages, stream: true}); +for await (const chunk of stream) conn.send(JSON.stringify({chunk: chunk.choices[0].delta.content})); +``` + +**Type-safe state:** `Agent` - third param types `conn.state` + +## MCP Integration + +Model Context Protocol for exposing tools: + +```ts +// Register & use MCP server +await this.mcp.registerServer("github", { + url: env.MCP_SERVER_URL, + auth: { type: "oauth", clientId: env.GITHUB_CLIENT_ID, clientSecret: env.GITHUB_CLIENT_SECRET } +}); +const tools = await this.mcp.getAITools(["github"]); +return this.streamText({ model: openai("gpt-4"), messages: this.messages, tools, onFinish }); +``` + +## Task Queue + +```ts +await this.queue("processVideo", { videoId: "abc123" }); // Add task +const tasks = await this.dequeue(10); // Process up to 10 +``` + +## Context & Cleanup + +```ts +const agent = getCurrentAgent(); // Get current instance +async destroy() { /* cleanup before agent destroyed */ } +``` + +## AI Integration + +```ts +// Workers AI +const r = await this.env.AI.run("@cf/meta/llama-3.1-8b-instruct", {prompt}); + +// Manual streaming (prefer AIChatAgent for auto-streaming) +const stream = await client.chat.completions.create({model: "gpt-4", messages, stream: true}); +for await (const chunk of stream) { + if (chunk.choices[0]?.delta?.content) conn.send(JSON.stringify({chunk: chunk.choices[0].delta.content})); +} +``` + +## Client Hooks (React) + +```ts +// useAgent() - WebSocket connection + RPC +import { useAgent } from "agents/react"; +const agent = useAgent({ agent: "MyAgent", name: "user-123" }); // name for idFromName +const result = await agent.processTask({ text: "Hello" }); // Call @callable methods +// agent.readyState: 0=CONNECTING, 1=OPEN, 2=CLOSING, 3=CLOSED + +// useAgentChat() - AI chat UI +import { useAgentChat } from "agents/ai-react"; +const agent = useAgent({ agent: "ChatAgent" }); +const { messages, input, handleInputChange, handleSubmit, isLoading, stop, clearHistory } = + useAgentChat({ + agent, + maxSteps: 5, // Max tool iterations + resume: true, // Auto-resume on disconnect + onToolCall: async (toolCall) => { + // Client tools (human-in-the-loop) + if (toolCall.toolName === "confirm") return { ok: window.confirm("Proceed?") }; + } + }); +// status: "ready" | "submitted" | "streaming" | "error" +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/agents-sdk/configuration.md b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/configuration.md new file mode 100644 index 0000000..a99cb03 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/configuration.md @@ -0,0 +1,182 @@ +# Configuration + +## Wrangler Setup + +```jsonc +{ + "name": "my-agents-app", + "durable_objects": { + "bindings": [ + {"name": "MyAgent", "class_name": "MyAgent"} + ] + }, + "migrations": [ + {"tag": "v1", "new_sqlite_classes": ["MyAgent"]} + ], + "ai": { + "binding": "AI" + } +} +``` + +## Environment Bindings + +**Type-safe pattern:** + +```typescript +interface Env { + AI?: Ai; // Workers AI + MyAgent?: DurableObjectNamespace; + ChatAgent?: DurableObjectNamespace; + DB?: D1Database; // D1 database + KV?: KVNamespace; // KV storage + R2?: R2Bucket; // R2 bucket + OPENAI_API_KEY?: string; // Secrets + GITHUB_CLIENT_ID?: string; // MCP OAuth credentials + GITHUB_CLIENT_SECRET?: string; + QUEUE?: Queue; // Queues +} +``` + +**Best practice:** Define all DO bindings in Env interface for type safety. + +## Deployment + +```bash +# Local dev +npx wrangler dev + +# Deploy production +npx wrangler deploy + +# Set secrets +npx wrangler secret put OPENAI_API_KEY +``` + +## Agent Routing + +**Recommended: Use route helpers** + +```typescript +import { routeAgent } from "agents"; + +export default { + fetch(request: Request, env: Env) { + return routeAgent(request, env); + } +} +``` + +Helper routes requests to agents automatically based on URL patterns. + +**Manual routing (advanced):** + +```typescript +export default { + async fetch(request: Request, env: Env) { + const url = new URL(request.url); + + // Named ID (deterministic) + const id = env.MyAgent.idFromName("user-123"); + + // Random ID (from URL param) + // const id = env.MyAgent.idFromString(url.searchParams.get("id")); + + const stub = env.MyAgent.get(id); + return stub.fetch(request); + } +} +``` + +**Multi-agent setup:** + +```typescript +import { routeAgent } from "agents"; + +export default { + fetch(request: Request, env: Env) { + const url = new URL(request.url); + + // Route by path + if (url.pathname.startsWith("/chat")) { + return routeAgent(request, env, "ChatAgent"); + } + if (url.pathname.startsWith("/task")) { + return routeAgent(request, env, "TaskAgent"); + } + + return new Response("Not found", { status: 404 }); + } +} +``` + +## Email Routing + +**Code setup:** + +```typescript +import { routeAgentEmail } from "agents"; + +export default { + fetch: (req: Request, env: Env) => routeAgent(req, env), + email: (message: ForwardableEmailMessage, env: Env) => { + return routeAgentEmail(message, env); + } +} +``` + +**Dashboard setup:** + +Configure email routing in Cloudflare dashboard: + +``` +Destination: Workers with Durable Objects +Worker: my-agents-app +``` + +Then handle in agent: + +```typescript +export class EmailAgent extends Agent { + async onEmail(email: AgentEmail) { + const text = await email.text(); + // Process email + } +} +``` + +## AI Gateway (Optional) + +```typescript +// Enable caching/routing through AI Gateway +const response = await this.env.AI.run( + "@cf/meta/llama-3.1-8b-instruct", + { prompt }, + { + gateway: { + id: "my-gateway-id", + skipCache: false, + cacheTtl: 3600 + } + } +); +``` + +## MCP Configuration (Optional) + +For exposing tools via Model Context Protocol: + +```typescript +// wrangler.jsonc - Add MCP OAuth secrets +{ + "vars": { + "MCP_SERVER_URL": "https://mcp.example.com" + } +} + +// Set secrets via CLI +// npx wrangler secret put GITHUB_CLIENT_ID +// npx wrangler secret put GITHUB_CLIENT_SECRET +``` + +Then register in agent code (see api.md MCP section). diff --git a/plugins/cloudflare/skills/cloudflare/references/agents-sdk/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/gotchas.md new file mode 100644 index 0000000..396bd1c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/gotchas.md @@ -0,0 +1,158 @@ +# Gotchas & Best Practices + +## Common Errors + +### "setState() not syncing" + +**Cause:** Mutating state directly or not calling `setState()` after modifications +**Solution:** Always use `setState()` with immutable updates: +```ts +// ❌ this.state.count++ +// ✅ this.setState({...this.state, count: this.state.count + 1}) +``` + +### "Message history grows unbounded (AIChatAgent)" + +**Cause:** `this.messages` in `AIChatAgent` accumulates all messages indefinitely +**Solution:** Manually trim old messages periodically: +```ts +export class ChatAgent extends AIChatAgent { + async onChatMessage(onFinish) { + // Keep only last 50 messages + if (this.messages.length > 50) { + this.messages = this.messages.slice(-50); + } + + return this.streamText({ model: openai("gpt-4"), messages: this.messages, onFinish }); + } +} +``` + +### "SQL injection vulnerability" + +**Cause:** Direct string interpolation in SQL queries +**Solution:** Use parameterized queries: +```ts +// ❌ this.sql`...WHERE id = '${userId}'` +// ✅ this.sql`...WHERE id = ${userId}` +``` + +### "WebSocket connection timeout" + +**Cause:** Not calling `conn.accept()` in `onConnect` +**Solution:** Always accept connections: +```ts +async onConnect(conn: Connection, ctx: ConnectionContext) { conn.accept(); conn.setState({userId: "123"}); } +``` + +### "Schedule limit exceeded" + +**Cause:** More than 1000 scheduled tasks per agent +**Solution:** Clean up old schedules and limit creation rate: +```ts +async checkSchedules() { if ((await this.getSchedules()).length > 800) console.warn("Near limit!"); } +``` + +### "AI Gateway unavailable" + +**Cause:** AI service timeout or quota exceeded +**Solution:** Add error handling and fallbacks: +```ts +try { + return await this.env.AI.run(model, {prompt}); +} catch (e) { + console.error("AI error:", e); + return {error: "Unavailable"}; +} +``` + +### "@callable method returns undefined" + +**Cause:** Method doesn't return JSON-serializable value, or has non-serializable types +**Solution:** Ensure return values are plain objects/arrays/primitives: +```ts +// ❌ Returns class instance +@callable() +async getData() { return new Date(); } + +// ✅ Returns serializable object +@callable() +async getData() { return { timestamp: Date.now() }; } +``` + +### "Resumable stream not resuming" + +**Cause:** Stream ID must be deterministic for resumption to work +**Solution:** Use AIChatAgent (automatic) or ensure consistent stream IDs: +```ts +// AIChatAgent handles this automatically +export class ChatAgent extends AIChatAgent { + // Resumption works out of the box +} +``` + +### "MCP connection loss on hibernation" + +**Cause:** MCP server connections don't survive hibernation +**Solution:** Re-register servers in `onStart()` or check connection status: +```ts +onStart() { + // Re-register MCP servers after hibernation + await this.mcp.registerServer("github", { url: env.MCP_URL, auth: {...} }); +} +``` + +### "Agent not found" + +**Cause:** Durable Object binding missing or incorrect class name +**Solution:** Verify DO binding in wrangler.jsonc and class name matches + +## Rate Limits & Quotas + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| CPU per request | 30s (std), 300s (max) | Set in wrangler.jsonc | +| Memory per instance | 128MB | Shared with WebSockets | +| Storage per agent | 10GB | SQLite storage | +| Scheduled tasks | 1000 per agent | Monitor with `getSchedules()` | +| WebSocket connections | Unlimited | Within memory limits | +| SQL columns | 100 | Per table | +| SQL row size | 2MB | Key + value | +| WebSocket message | 32MiB | Max size | +| DO requests/sec | ~1000 | Per unique DO instance; rate limit if needed | +| AI Gateway (Workers AI) | Model-specific | Check dashboard for limits | +| MCP requests | Depends on server | Implement retry/backoff | + +## Best Practices + +### State Management +- Use immutable updates: `setState({...this.state, key: newValue})` +- Trim unbounded arrays (messages, logs) periodically +- Store large data in SQL, not state + +### SQL Usage +- Create tables in `onStart()`, not `onRequest()` +- Use parameterized queries: `` sql`WHERE id = ${id}` `` (NOT `` sql`WHERE id = '${id}'` ``) +- Index frequently queried columns + +### Scheduling +- Monitor schedule count: `await this.getSchedules()` +- Cancel completed tasks to stay under 1000 limit +- Use cron strings for recurring tasks + +### WebSockets +- Always call `conn.accept()` in `onConnect()` +- Handle client disconnects gracefully +- Broadcast to `this.connections` efficiently + +### AI Integration +- Use `AIChatAgent` for chat interfaces (auto-streaming, resumption) +- Trim message history to avoid token limits +- Handle AI errors with try/catch and fallbacks + +### Production Deployment +- **Rate limiting:** Implement request throttling for high-traffic agents (>1000 req/s) +- **Monitoring:** Log critical errors, track schedule count, monitor storage usage +- **Graceful degradation:** Handle AI service outages with fallbacks +- **Message trimming:** Enforce max history length (e.g., 100 messages) in AIChatAgent +- **MCP reliability:** Re-register servers on hibernation, implement retry logic diff --git a/plugins/cloudflare/skills/cloudflare/references/agents-sdk/patterns.md b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/patterns.md new file mode 100644 index 0000000..2e72d4b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/agents-sdk/patterns.md @@ -0,0 +1,192 @@ +# Patterns & Use Cases + +## AI Chat w/Tools + +**Server (AIChatAgent):** + +```ts +import { AIChatAgent } from "agents"; +import { openai } from "@ai-sdk/openai"; +import { tool } from "ai"; +import { z } from "zod"; + +export class ChatAgent extends AIChatAgent { + async onChatMessage(onFinish) { + return this.streamText({ + model: openai("gpt-4"), + messages: this.messages, // Auto-managed + tools: { + getWeather: tool({ + description: "Get current weather", + parameters: z.object({ city: z.string() }), + execute: async ({ city }) => `Weather in ${city}: Sunny, 72°F` + }), + searchDocs: tool({ + description: "Search documentation", + parameters: z.object({ query: z.string() }), + execute: async ({ query }) => JSON.stringify( + this.sql<{title, content}>`SELECT title, content FROM docs WHERE content LIKE ${'%' + query + '%'}` + ) + }) + }, + onFinish, + }); + } +} +``` + +**Client (React):** + +```tsx +import { useAgent } from "agents/react"; +import { useAgentChat } from "agents/ai-react"; + +function ChatUI() { + const agent = useAgent({ agent: "ChatAgent" }); + const { messages, input, handleInputChange, handleSubmit, isLoading } = useAgentChat({ agent }); + + return ( +
+ {messages.map(m =>
{m.role}: {m.content}
)} +
+ + +
+
+ ); +} +``` + +## Human-in-the-Loop (Client Tools) + +Server defines tool, client executes: + +```ts +// Server +export class ChatAgent extends AIChatAgent { + async onChatMessage(onFinish) { + return this.streamText({ + model: openai("gpt-4"), + messages: this.messages, + tools: { + confirmAction: tool({ + description: "Ask user to confirm", + parameters: z.object({ action: z.string() }), + execute: "client", // Client-side execution + }) + }, + onFinish, + }); + } +} + +// Client +const { messages } = useAgentChat({ + agent, + onToolCall: async (toolCall) => { + if (toolCall.toolName === "confirmAction") { + return { confirmed: window.confirm(`Confirm: ${toolCall.args.action}?`) }; + } + } +}); +``` + +## Task Queue & Scheduled Processing + +```ts +export class TaskAgent extends Agent { + onStart() { + this.schedule("*/5 * * * *", "processQueue", {}); // Every 5 min + this.schedule("0 0 * * *", "dailyCleanup", {}); // Daily + } + + async onRequest(req: Request) { + await this.queue("processVideo", { videoId: (await req.json()).videoId }); + return Response.json({ queued: true }); + } + + async processQueue() { + const tasks = await this.dequeue(10); + for (const task of tasks) { + if (task.name === "processVideo") await this.processVideo(task.data.videoId); + } + } + + async dailyCleanup() { + this.sql`DELETE FROM logs WHERE created_at < ${Date.now() - 86400000}`; + } +} +``` + +## Manual WebSocket Chat + +Custom protocols (non-AI): + +```ts +export class ChatAgent extends Agent { + async onConnect(conn: Connection, ctx: ConnectionContext) { + conn.accept(); + conn.setState({userId: ctx.request.headers.get("X-User-ID") || "anon"}); + conn.send(JSON.stringify({type: "history", messages: this.state.messages})); + } + + async onMessage(conn: Connection, msg: WSMessage) { + const newMsg = {userId: conn.state.userId, text: JSON.parse(msg as string).text, timestamp: Date.now()}; + this.setState({messages: [...this.state.messages, newMsg]}); + this.connections.forEach(c => c.send(JSON.stringify(newMsg))); + } +} +``` + +## Email Processing w/AI + +```ts +export class EmailAgent extends Agent { + async onEmail(email: AgentEmail) { + const [text, from, subject] = [await email.text(), email.from, email.headers.get("subject") || ""]; + this.sql`INSERT INTO emails (from_addr, subject, body) VALUES (${from}, ${subject}, ${text})`; + + const { text: summary } = await generateText({ + model: openai("gpt-4o-mini"), prompt: `Summarize: ${subject}\n\n${text}` + }); + + this.connections.forEach(c => c.send(JSON.stringify({type: "new_email", from, summary}))); + if (summary.includes("urgent")) await this.schedule(0, "sendAutoReply", { to: from }); + } +} +``` + +## Real-time Collaboration + +```ts +export class GameAgent extends Agent { + initialState = { players: [], gameStarted: false }; + + async onConnect(conn: Connection, ctx: ConnectionContext) { + conn.accept(); + const playerId = ctx.request.headers.get("X-Player-ID") || crypto.randomUUID(); + conn.setState({ playerId }); + + const newPlayer = { id: playerId, score: 0 }; + this.setState({...this.state, players: [...this.state.players, newPlayer]}); + this.connections.forEach(c => c.send(JSON.stringify({type: "player_joined", player: newPlayer}))); + } + + async onMessage(conn: Connection, msg: WSMessage) { + const m = JSON.parse(msg as string); + + if (m.type === "move") { + this.setState({ + ...this.state, + players: this.state.players.map(p => p.id === conn.state.playerId ? {...p, score: p.score + m.points} : p) + }); + this.connections.forEach(c => c.send(JSON.stringify({type: "player_moved", playerId: conn.state.playerId}))); + } + + if (m.type === "start" && this.state.players.length >= 2) { + this.setState({...this.state, gameStarted: true}); + this.connections.forEach(c => c.send(JSON.stringify({type: "game_started"}))); + } + } +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-gateway/README.md b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/README.md new file mode 100644 index 0000000..75a12ab --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/README.md @@ -0,0 +1,175 @@ +# Cloudflare AI Gateway + +Expert guidance for implementing Cloudflare AI Gateway - a universal gateway for AI model providers with analytics, caching, rate limiting, and routing capabilities. + +## When to Use This Reference + +- Setting up AI Gateway for any AI provider (OpenAI, Anthropic, Workers AI, etc.) +- Implementing caching, rate limiting, or request retry/fallback +- Configuring dynamic routing with A/B testing or model fallbacks +- Managing provider API keys securely with BYOK +- Adding security features (guardrails, DLP) +- Setting up observability with logging and custom metadata +- Debugging AI Gateway requests or optimizing configurations + +## Quick Start + +**What's your setup?** + +- **Using Vercel AI SDK** → Pattern 1 (recommended) - see [sdk-integration.md](./sdk-integration.md) +- **Using OpenAI SDK** → Pattern 2 - see [sdk-integration.md](./sdk-integration.md) +- **Cloudflare Worker + Workers AI** → Pattern 3 - see [sdk-integration.md](./sdk-integration.md) +- **Direct HTTP (any language)** → Pattern 4 - see [configuration.md](./configuration.md) +- **Framework (LangChain, etc.)** → See [sdk-integration.md](./sdk-integration.md) + +## Pattern 1: Vercel AI SDK (Recommended) + +Most modern pattern using official `ai-gateway-provider` package with automatic fallbacks. + +```typescript +import { createAiGateway } from 'ai-gateway-provider'; +import { createOpenAI } from '@ai-sdk/openai'; +import { generateText } from 'ai'; + +const gateway = createAiGateway({ + accountId: process.env.CF_ACCOUNT_ID, + gateway: process.env.CF_GATEWAY_ID, +}); + +const openai = createOpenAI({ + apiKey: process.env.OPENAI_API_KEY +}); + +// Single model +const { text } = await generateText({ + model: gateway(openai('gpt-4o')), + prompt: 'Hello' +}); + +// Automatic fallback array +const { text } = await generateText({ + model: gateway([ + openai('gpt-4o'), // Try first + anthropic('claude-sonnet-4-5'), // Fallback + ]), + prompt: 'Hello' +}); +``` + +**Install:** `npm install ai-gateway-provider ai @ai-sdk/openai @ai-sdk/anthropic` + +## Pattern 2: OpenAI SDK + +Drop-in replacement for OpenAI API with multi-provider support. + +```typescript +import OpenAI from 'openai'; + +const client = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/compat`, + defaultHeaders: { + 'cf-aig-authorization': `Bearer ${cfToken}` // For authenticated gateways + } +}); + +// Switch providers by changing model format: {provider}/{model} +const response = await client.chat.completions.create({ + model: 'openai/gpt-4o', // or 'anthropic/claude-sonnet-4-5' + messages: [{ role: 'user', content: 'Hello!' }] +}); +``` + +## Pattern 3: Workers AI Binding + +For Cloudflare Workers using Workers AI. + +```typescript +export default { + async fetch(request, env, ctx) { + const response = await env.AI.run( + '@cf/meta/llama-3-8b-instruct', + { messages: [{ role: 'user', content: 'Hello!' }] }, + { + gateway: { + id: 'my-gateway', + metadata: { userId: '123', team: 'engineering' } + } + } + ); + + return Response.json(response); + } +}; +``` + +## Headers Quick Reference + +| Header | Purpose | Example | Notes | +|--------|---------|---------|-------| +| `cf-aig-authorization` | Gateway auth | `Bearer {token}` | Required for authenticated gateways | +| `cf-aig-metadata` | Tracking | `{"userId":"x"}` | Max 5 entries, flat structure | +| `cf-aig-cache-ttl` | Cache duration | `3600` | Seconds, min 60, max 2592000 (30 days) | +| `cf-aig-skip-cache` | Bypass cache | `true` | - | +| `cf-aig-cache-key` | Custom cache key | `my-key` | Must be unique per response | +| `cf-aig-collect-log` | Skip logging | `false` | Default: true | +| `cf-aig-cache-status` | Cache hit/miss | Response only | `HIT` or `MISS` | + +## In This Reference + +| File | Purpose | +|------|---------| +| [sdk-integration.md](./sdk-integration.md) | Vercel AI SDK, OpenAI SDK, Workers binding patterns | +| [configuration.md](./configuration.md) | Dashboard setup, wrangler, API tokens | +| [features.md](./features.md) | Caching, rate limits, guardrails, DLP, BYOK, unified billing | +| [dynamic-routing.md](./dynamic-routing.md) | Fallbacks, A/B testing, conditional routing | +| [troubleshooting.md](./troubleshooting.md) | Debugging, errors, observability, gotchas | + +## Reading Order + +| Task | Files | +|------|-------| +| First-time setup | README + [configuration.md](./configuration.md) | +| SDK integration | README + [sdk-integration.md](./sdk-integration.md) | +| Enable caching | README + [features.md](./features.md) | +| Setup fallbacks | README + [dynamic-routing.md](./dynamic-routing.md) | +| Debug errors | README + [troubleshooting.md](./troubleshooting.md) | + +## Architecture + +AI Gateway acts as a proxy between your application and AI providers: + +``` +Your App → AI Gateway → AI Provider (OpenAI, Anthropic, etc.) + ↓ + Analytics, Caching, Rate Limiting, Logging +``` + +**Key URL patterns:** +- Unified API (OpenAI-compatible): `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/compat/chat/completions` +- Provider-specific: `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/{provider}/{endpoint}` +- Dynamic routes: Use route name instead of model: `dynamic/{route-name}` + +## Gateway Types + +1. **Unauthenticated Gateway**: Open access (not recommended for production) +2. **Authenticated Gateway**: Requires `cf-aig-authorization` header with Cloudflare API token (recommended) + +## Provider Authentication Options + +1. **Unified Billing**: Use AI Gateway billing to pay for inference (keyless mode - no provider API key needed) +2. **BYOK (Store Keys)**: Store provider API keys in Cloudflare dashboard +3. **Request Headers**: Include provider API key in each request + +## Related Skills + +- [Workers AI](../workers-ai/README.md) - For `env.AI.run()` details +- [Agents SDK](../agents-sdk/README.md) - For stateful AI patterns +- [Vectorize](../vectorize/README.md) - For RAG patterns with embeddings + +## Resources + +- [Official Docs](https://developers.cloudflare.com/ai-gateway/) +- [API Reference](https://developers.cloudflare.com/api/resources/ai_gateway/) +- [Provider Guides](https://developers.cloudflare.com/ai-gateway/usage/providers/) +- [Discord Community](https://discord.cloudflare.com) diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-gateway/configuration.md b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/configuration.md new file mode 100644 index 0000000..78b5615 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/configuration.md @@ -0,0 +1,111 @@ +# Configuration & Setup + +## Creating a Gateway + +### Dashboard +AI > AI Gateway > Create Gateway > Configure (auth, caching, rate limiting, logging) + +### API +```bash +curl -X POST https://api.cloudflare.com/client/v4/accounts/{account_id}/ai-gateway/gateways \ + -H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" \ + -d '{"id":"my-gateway","cache_ttl":3600,"rate_limiting_interval":60,"rate_limiting_limit":100,"collect_logs":true}' +``` + +**Naming:** lowercase alphanumeric + hyphens (e.g., `prod-api`, `dev-chat`) + +## Wrangler Integration + +```toml +[ai] +binding = "AI" + +[[ai.gateway]] +id = "my-gateway" +``` + +```bash +wrangler secret put CF_API_TOKEN +wrangler secret put OPENAI_API_KEY # If not using BYOK +``` + +## Authentication + +### Gateway Auth (protects gateway access) +```typescript +const client = new OpenAI({ + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/openai`, + defaultHeaders: { 'cf-aig-authorization': `Bearer ${cfToken}` } +}); +``` + +### Provider Auth Options + +**1. Unified Billing (keyless)** - pay through Cloudflare, no provider key: +```typescript +const client = new OpenAI({ + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/openai`, + defaultHeaders: { 'cf-aig-authorization': `Bearer ${cfToken}` } +}); +``` +Supports: OpenAI, Anthropic, Google AI Studio + +**2. BYOK** - store keys in dashboard (Provider Keys > Add), no key in code + +**3. Request Headers** - pass provider key per request: +```typescript +const client = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/openai`, + defaultHeaders: { 'cf-aig-authorization': `Bearer ${cfToken}` } +}); +``` + +## API Token Permissions + +- **Gateway management:** AI Gateway - Read + Edit +- **Gateway access:** AI Gateway - Read (minimum) + +## Gateway Management API + +```bash +# List +curl https://api.cloudflare.com/client/v4/accounts/{account_id}/ai-gateway/gateways \ + -H "Authorization: Bearer $CF_API_TOKEN" + +# Get +curl .../gateways/{gateway_id} + +# Update +curl -X PUT .../gateways/{gateway_id} \ + -d '{"cache_ttl":7200,"rate_limiting_limit":200}' + +# Delete +curl -X DELETE .../gateways/{gateway_id} +``` + +## Getting IDs + +- **Account ID:** Dashboard > Overview > Copy +- **Gateway ID:** AI Gateway > Gateway name column + +## Python Example + +```python +from openai import OpenAI +import os + +client = OpenAI( + api_key=os.environ.get("OPENAI_API_KEY"), + base_url=f"https://gateway.ai.cloudflare.com/v1/{os.environ['CF_ACCOUNT_ID']}/{os.environ['GATEWAY_ID']}/openai", + default_headers={"cf-aig-authorization": f"Bearer {os.environ['CF_API_TOKEN']}"} +) +``` + +## Best Practices + +1. **Always authenticate gateways in production** +2. **Use BYOK or unified billing** - secrets out of code +3. **Environment-specific gateways** - separate dev/staging/prod +4. **Set rate limits** - prevent runaway costs +5. **Enable logging** - track usage, debug issues diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-gateway/dynamic-routing.md b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/dynamic-routing.md new file mode 100644 index 0000000..540be5c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/dynamic-routing.md @@ -0,0 +1,82 @@ +# Dynamic Routing + +Configure complex routing in dashboard without code changes. Use route names instead of model names. + +## Usage + +```typescript +const response = await client.chat.completions.create({ + model: 'dynamic/smart-chat', // Route name from dashboard + messages: [{ role: 'user', content: 'Hello!' }] +}); +``` + +## Node Types + +| Node | Purpose | Use Case | +|------|---------|----------| +| **Conditional** | Branch on metadata | Paid vs free users, geo routing | +| **Percentage** | A/B split traffic | Model testing, gradual rollouts | +| **Rate Limit** | Enforce quotas | Per-user/team limits | +| **Budget Limit** | Cost quotas | Per-user spending caps | +| **Model** | Call provider | Final destination | + +## Metadata + +Pass via header (max 5 entries, flat only): +```typescript +headers: { + 'cf-aig-metadata': JSON.stringify({ + userId: 'user-123', + tier: 'pro', + region: 'us-east' + }) +} +``` + +## Common Patterns + +**Multi-model fallback:** +``` +Start → GPT-4 → On error: Claude → On error: Llama +``` + +**Tiered access:** +``` +Conditional: tier == 'enterprise' → GPT-4 (no limit) +Conditional: tier == 'pro' → Rate Limit 1000/hr → GPT-4o +Conditional: tier == 'free' → Rate Limit 10/hr → GPT-4o-mini +``` + +**Gradual rollout:** +``` +Percentage: 10% → New model, 90% → Old model +``` + +**Cost-based fallback:** +``` +Budget Limit: $100/day per teamId + < 80%: GPT-4 + >= 80%: GPT-4o-mini + >= 100%: Error +``` + +## Version Management + +- Save changes as new version +- Test with `model: 'dynamic/route@v2'` +- Roll back by deploying previous version + +## Monitoring + +Dashboard → Gateway → Dynamic Routes: +- Request count per path +- Success/error rates +- Latency/cost by path + +## Limitations + +- Max 5 metadata entries +- Values: string/number/boolean/null only +- No nested objects +- Route names: alphanumeric + hyphens diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-gateway/features.md b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/features.md new file mode 100644 index 0000000..4a0d384 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/features.md @@ -0,0 +1,96 @@ +# Features & Capabilities + +## Caching + +Dashboard: Settings → Cache Responses → Enable + +```typescript +// Custom TTL (1 hour) +headers: { 'cf-aig-cache-ttl': '3600' } + +// Skip cache +headers: { 'cf-aig-skip-cache': 'true' } + +// Custom cache key +headers: { 'cf-aig-cache-key': 'greeting-en' } +``` + +**Limits:** TTL 60s - 30 days. **Does NOT work with streaming.** + +## Rate Limiting + +Dashboard: Settings → Rate-limiting → Enable + +- **Fixed window:** Resets at intervals +- **Sliding window:** Rolling window (more accurate) +- Returns `429` when exceeded + +## Guardrails + +Dashboard: Settings → Guardrails → Enable + +Filter prompts/responses for inappropriate content. Actions: Flag (log) or Block (reject). + +## Data Loss Prevention (DLP) + +Dashboard: Settings → DLP → Enable + +Detect PII (emails, SSNs, credit cards). Actions: Flag, Block, or Redact. + +## Billing Modes + +| Mode | Description | Setup | +|------|-------------|-------| +| **Unified Billing** | Pay through Cloudflare, no provider keys | Use `cf-aig-authorization` header only | +| **BYOK** | Store provider keys in dashboard | Add keys in Provider Keys section | +| **Pass-through** | Send provider key with each request | Include provider's auth header | + +## Zero Data Retention + +Dashboard: Settings → Privacy → Zero Data Retention + +No prompts/responses stored. Request counts and costs still tracked. + +## Logging + +Dashboard: Settings → Logs → Enable (up to 10M logs) + +Each entry: prompt, response, provider, model, tokens, cost, duration, cache status, metadata. + +```typescript +// Skip logging for request +headers: { 'cf-aig-collect-log': 'false' } +``` + +**Export:** Use Logpush to S3, GCS, Datadog, Splunk, etc. + +## Custom Cost Tracking + +For models not in Cloudflare's pricing database: + +Dashboard: Gateway → Settings → Custom Costs + +Or via API: set `model`, `input_cost`, `output_cost`. + +## Supported Providers (22+) + +| Provider | Unified API | Notes | +|----------|-------------|-------| +| OpenAI | `openai/gpt-4o` | Full support | +| Anthropic | `anthropic/claude-sonnet-4-5` | Full support | +| Google AI | `google-ai-studio/gemini-2.0-flash` | Full support | +| Workers AI | `workersai/@cf/meta/llama-3` | Native | +| Azure OpenAI | `azure-openai/*` | Deployment names | +| AWS Bedrock | Provider endpoint only | `/bedrock/*` | +| Groq | `groq/*` | Fast inference | +| Mistral, Cohere, Perplexity, xAI, DeepSeek, Cerebras | Full support | - | + +## Best Practices + +1. Enable caching for deterministic prompts +2. Set rate limits to prevent abuse +3. Use guardrails for user-facing AI +4. Enable DLP for sensitive data +5. Use unified billing or BYOK for simpler key management +6. Enable logging for debugging +7. Use zero data retention when privacy required diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-gateway/sdk-integration.md b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/sdk-integration.md new file mode 100644 index 0000000..7ca4930 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/sdk-integration.md @@ -0,0 +1,114 @@ +# AI Gateway SDK Integration + +## Vercel AI SDK (Recommended) + +```typescript +import { createAiGateway } from 'ai-gateway-provider'; +import { createOpenAI } from '@ai-sdk/openai'; +import { generateText } from 'ai'; + +const gateway = createAiGateway({ + accountId: process.env.CF_ACCOUNT_ID, + gateway: process.env.CF_GATEWAY_ID, + apiKey: process.env.CF_API_TOKEN // Optional for auth gateways +}); + +const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY }); + +// Single model +const { text } = await generateText({ + model: gateway(openai('gpt-4o')), + prompt: 'Hello' +}); + +// Automatic fallback array +const { text } = await generateText({ + model: gateway([ + openai('gpt-4o'), + anthropic('claude-sonnet-4-5'), + openai('gpt-4o-mini') + ]), + prompt: 'Complex task' +}); +``` + +### Options + +```typescript +model: gateway(openai('gpt-4o'), { + cacheKey: 'my-key', + cacheTtl: 3600, + metadata: { userId: 'u123', team: 'eng' }, // Max 5 entries + retries: { maxAttempts: 3, backoff: 'exponential' } +}) +``` + +## OpenAI SDK + +```typescript +const client = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/openai`, + defaultHeaders: { 'cf-aig-authorization': `Bearer ${cfToken}` } +}); + +// Unified API - switch providers via model name +model: 'openai/gpt-4o' // or 'anthropic/claude-sonnet-4-5' +``` + +## Anthropic SDK + +```typescript +const client = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/anthropic`, + defaultHeaders: { 'cf-aig-authorization': `Bearer ${cfToken}` } +}); +``` + +## Workers AI Binding + +```toml +# wrangler.toml +[ai] +binding = "AI" +[[ai.gateway]] +id = "my-gateway" +``` + +```typescript +await env.AI.run('@cf/meta/llama-3-8b-instruct', + { messages: [...] }, + { gateway: { id: 'my-gateway', metadata: { userId: '123' } } } +); +``` + +## LangChain / LlamaIndex + +```typescript +// Use OpenAI SDK pattern with custom baseURL +new ChatOpenAI({ + configuration: { + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/openai` + } +}); +``` + +## HTTP / cURL + +```bash +curl https://gateway.ai.cloudflare.com/v1/{account}/{gateway}/openai/chat/completions \ + -H "Authorization: Bearer $OPENAI_KEY" \ + -H "cf-aig-authorization: Bearer $CF_TOKEN" \ + -H "cf-aig-metadata: {\"userId\":\"123\"}" \ + -d '{"model":"gpt-4o","messages":[...]}' +``` + +## Headers Reference + +| Header | Purpose | +|--------|---------| +| `cf-aig-authorization` | Gateway auth token | +| `cf-aig-metadata` | JSON object (max 5 keys) | +| `cf-aig-cache-ttl` | Cache TTL in seconds | +| `cf-aig-skip-cache` | `true` to bypass cache | diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-gateway/troubleshooting.md b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/troubleshooting.md new file mode 100644 index 0000000..4d66357 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-gateway/troubleshooting.md @@ -0,0 +1,88 @@ +# AI Gateway Troubleshooting + +## Common Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| 401 | Missing `cf-aig-authorization` header | Add header with CF API token | +| 403 | Invalid provider key / BYOK expired | Check provider key in dashboard | +| 429 | Rate limit exceeded | Increase limit or implement backoff | + +### 401 Fix + +```typescript +const client = new OpenAI({ + baseURL: `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/openai`, + defaultHeaders: { 'cf-aig-authorization': `Bearer ${CF_API_TOKEN}` } +}); +``` + +### 429 Retry Pattern + +```typescript +async function requestWithRetry(fn, maxRetries = 3) { + for (let i = 0; i < maxRetries; i++) { + try { return await fn(); } + catch (e) { + if (e.status === 429 && i < maxRetries - 1) { + await new Promise(r => setTimeout(r, Math.pow(2, i) * 1000)); + continue; + } + throw e; + } + } +} +``` + +## Gotchas + +| Issue | Reality | +|-------|---------| +| Metadata limits | Max 5 entries, flat only (no nesting) | +| Cache key collision | Use unique keys per expected response | +| BYOK + Unified Billing | Mutually exclusive | +| Rate limit scope | Per-gateway, not per-user (use dynamic routing for per-user) | +| Log delay | 30-60 seconds normal | +| Streaming + caching | **Incompatible** | +| Model name (unified API) | Prefix required: `openai/gpt-4o`, not `gpt-4o` | + +## Cache Not Working + +**Causes:** +- Different request params (temperature, etc.) +- Streaming enabled +- Caching disabled in settings + +**Check:** `response.headers.get('cf-aig-cache-status')` → HIT or MISS + +## Logs Not Appearing + +1. Check logging enabled: Dashboard → Gateway → Settings +2. Remove `cf-aig-collect-log: false` header +3. Wait 30-60 seconds +4. Check log limit (10M default) + +## Debugging + +```bash +# Test connectivity +curl -v https://gateway.ai.cloudflare.com/v1/{account}/{gateway}/openai/models \ + -H "Authorization: Bearer $OPENAI_KEY" \ + -H "cf-aig-authorization: Bearer $CF_TOKEN" +``` + +```typescript +// Check response headers +console.log('Cache:', response.headers.get('cf-aig-cache-status')); +console.log('Request ID:', response.headers.get('cf-ray')); +``` + +## Analytics + +Dashboard → AI Gateway → Select gateway + +**Metrics:** Requests, tokens, latency (p50/p95/p99), cache hit rate, costs + +**Log filters:** `status: error`, `provider: openai`, `cost > 0.01`, `duration > 1000` + +**Export:** Logpush to S3/GCS/Datadog/Splunk diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-search/README.md b/plugins/cloudflare/skills/cloudflare/references/ai-search/README.md new file mode 100644 index 0000000..52d766d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-search/README.md @@ -0,0 +1,138 @@ +# Cloudflare AI Search Reference + +Expert guidance for implementing Cloudflare AI Search (formerly AutoRAG), Cloudflare's managed semantic search and RAG service. + +## Overview + +**AI Search** is a managed RAG (Retrieval-Augmented Generation) pipeline that combines: +- Automatic semantic indexing of your content +- Vector similarity search +- Built-in LLM generation + +**Key value propositions:** +- **Zero vector management** - No manual embedding, indexing, or storage +- **Auto-indexing** - Content automatically re-indexed every 6 hours +- **Built-in generation** - Optional AI response generation from retrieved context +- **Multi-source** - Index from R2 buckets or website crawls + +**Data source options:** +- **R2 bucket** - Index files from Cloudflare R2 (supports MD, TXT, HTML, PDF, DOC, CSV, JSON) +- **Website** - Crawl and index website content (requires Cloudflare-hosted domain) + +**Indexing lifecycle:** +- Automatic 6-hour refresh cycle +- Manual "Force Sync" available (30s rate limit) +- Not designed for real-time updates + +## Quick Start + +**1. Create AI Search instance in dashboard:** +- Go to Cloudflare Dashboard → AI Search → Create +- Choose data source (R2 or website) +- Configure instance name and settings + +**2. Configure Worker:** + +```jsonc +// wrangler.jsonc +{ + "ai": { + "binding": "AI" + } +} +``` + +**3. Use in Worker:** + +```typescript +export default { + async fetch(request, env) { + const answer = await env.AI.autorag("my-search-instance").aiSearch({ + query: "How do I configure caching?", + model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast" + }); + + return Response.json({ answer: answer.response }); + } +}; +``` + +## When to Use AI Search + +### AI Search vs Vectorize + +| Factor | AI Search | Vectorize | +|--------|-----------|-----------| +| **Management** | Fully managed | Manual embedding + indexing | +| **Use when** | Want zero-ops RAG pipeline | Need custom embeddings/control | +| **Indexing** | Automatic (6hr cycle) | Manual via API | +| **Generation** | Built-in optional | Bring your own LLM | +| **Data sources** | R2 or website | Manual insert | +| **Best for** | Docs, support, enterprise search | Custom ML pipelines, real-time | + +### AI Search vs Direct Workers AI + +| Factor | AI Search | Workers AI (direct) | +|--------|-----------|---------------------| +| **Context** | Automatic retrieval | Manual context building | +| **Use when** | Need RAG (search + generate) | Simple generation tasks | +| **Indexing** | Built-in | Not applicable | +| **Best for** | Knowledge bases, docs | Simple chat, transformations | + +### search() vs aiSearch() + +| Method | Returns | Use When | +|--------|---------|----------| +| `search()` | Search results only | Building custom UI, need raw chunks | +| `aiSearch()` | AI response + results | Need ready-to-use answer (chatbot, Q&A) | + +### Real-time Updates Consideration + +**AI Search is NOT ideal if:** +- Need real-time content updates (<6 hours) +- Content changes multiple times per hour +- Strict freshness requirements + +**AI Search IS ideal if:** +- Content relatively stable (docs, policies, knowledge bases) +- 6-hour refresh acceptable +- Prefer zero-ops over real-time + +## Platform Limits + +| Limit | Value | +|-------|-------| +| Max instances per account | 10 | +| Max files per instance | 100,000 | +| Max file size | 4 MB | +| Index frequency | Every 6 hours | +| Force Sync rate limit | Once per 30 seconds | +| Filter nesting depth | 2 levels | +| Filters per compound | 10 | +| Score threshold range | 0.0 - 1.0 | + +## Reading Order + +Navigate these references based on your task: + +| Task | Read | Est. Time | +|------|------|-----------| +| **Understand AI Search** | README only | 5 min | +| **Implement basic search** | README → api.md | 10 min | +| **Configure data source** | README → configuration.md | 10 min | +| **Production patterns** | patterns.md | 15 min | +| **Debug issues** | gotchas.md | 10 min | +| **Full implementation** | README → api.md → patterns.md | 30 min | + +## In This Reference + +- **[api.md](api.md)** - API endpoints, methods, TypeScript interfaces +- **[configuration.md](configuration.md)** - Setup, data sources, wrangler config +- **[patterns.md](patterns.md)** - Common patterns, decision guidance, code examples +- **[gotchas.md](gotchas.md)** - Troubleshooting, code-level gotchas, limits + +## See Also + +- [Cloudflare AI Search Docs](https://developers.cloudflare.com/ai-search/) +- [Workers AI Docs](https://developers.cloudflare.com/workers-ai/) +- [Vectorize Docs](https://developers.cloudflare.com/vectorize/) diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-search/api.md b/plugins/cloudflare/skills/cloudflare/references/ai-search/api.md new file mode 100644 index 0000000..b6220c4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-search/api.md @@ -0,0 +1,87 @@ +# AI Search API Reference + +## Workers Binding + +```typescript +const answer = await env.AI.autorag("instance-name").aiSearch(options); +const results = await env.AI.autorag("instance-name").search(options); +const instances = await env.AI.autorag("_").listInstances(); +``` + +## aiSearch() Options + +```typescript +interface AiSearchOptions { + query: string; // User query + model: string; // Workers AI model ID + system_prompt?: string; // LLM instructions + rewrite_query?: boolean; // Fix typos (default: false) + max_num_results?: number; // Max chunks (default: 10) + ranking_options?: { score_threshold?: number }; // 0.0-1.0 (default: 0.3) + reranking?: { enabled: boolean; model: string }; + stream?: boolean; // Stream response (default: false) + filters?: Filter; // Metadata filters + page?: string; // Pagination token +} +``` + +## Response + +```typescript +interface AiSearchResponse { + search_query: string; // Query used (rewritten if enabled) + response: string; // AI-generated answer + data: SearchResult[]; // Retrieved chunks + has_more: boolean; + next_page?: string; +} + +interface SearchResult { + id: string; + score: number; + content: string; + metadata: { filename: string; folder: string; timestamp: number }; +} +``` + +## Filters + +```typescript +// Comparison +{ column: "folder", operator: "gte", value: "docs/" } + +// Compound +{ operator: "and", filters: [ + { column: "folder", operator: "gte", value: "docs/" }, + { column: "timestamp", operator: "gte", value: 1704067200 } +]} +``` + +**Operators:** `eq`, `ne`, `gt`, `gte`, `lt`, `lte` + +**Built-in metadata:** `filename`, `folder`, `timestamp` (Unix seconds) + +## Streaming + +```typescript +const stream = await env.AI.autorag("docs").aiSearch({ query, model, stream: true }); +return new Response(stream, { headers: { "Content-Type": "text/event-stream" } }); +``` + +## Error Types + +| Error | Cause | +|-------|-------| +| `AutoRAGNotFoundError` | Instance doesn't exist | +| `AutoRAGUnauthorizedError` | Invalid/missing token | +| `AutoRAGValidationError` | Invalid parameters | + +## REST API + +```bash +curl https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/autorag/rags/{NAME}/ai-search \ + -H "Authorization: Bearer {TOKEN}" \ + -d '{"query": "...", "model": "@cf/meta/llama-3.3-70b-instruct-fp8-fast"}' +``` + +Requires Service API token with "AI Search - Read" permission. diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-search/configuration.md b/plugins/cloudflare/skills/cloudflare/references/ai-search/configuration.md new file mode 100644 index 0000000..d1f34ad --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-search/configuration.md @@ -0,0 +1,88 @@ +# AI Search Configuration + +## Worker Setup + +```jsonc +// wrangler.jsonc +{ + "ai": { "binding": "AI" } +} +``` + +```typescript +interface Env { + AI: Ai; +} + +const answer = await env.AI.autorag("my-instance").aiSearch({ + query: "How do I configure caching?", + model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast" +}); +``` + +## Data Sources + +### R2 Bucket + +Dashboard: AI Search → Create Instance → Select R2 bucket + +**Supported formats:** `.md`, `.txt`, `.html`, `.pdf`, `.doc`, `.docx`, `.csv`, `.json` + +**Auto-indexed metadata:** `filename`, `folder`, `timestamp` + +### Website Crawler + +Requirements: +- Domain on Cloudflare +- `sitemap.xml` at root +- Bot protection must allow `CloudflareAISearch` user agent + +## Path Filtering (R2) + +``` +docs/**/*.md # All .md in docs/ recursively +**/*.draft.md # Exclude (use in exclude patterns) +``` + +## Indexing + +- **Automatic:** Every 6 hours +- **Force Sync:** Dashboard button (30s rate limit between syncs) +- **Pause:** Settings → Pause Indexing (existing index remains searchable) + +## Service API Token + +Dashboard: AI Search → Instance → Use AI Search → API → Create Token + +Permissions: +- **Read** - search operations +- **Edit** - instance management + +Store securely: +```bash +wrangler secret put AI_SEARCH_TOKEN +``` + +## Multi-Environment + +```toml +# wrangler.toml +[env.production.vars] +AI_SEARCH_INSTANCE = "prod-docs" + +[env.staging.vars] +AI_SEARCH_INSTANCE = "staging-docs" +``` + +```typescript +const answer = await env.AI.autorag(env.AI_SEARCH_INSTANCE).aiSearch({ query }); +``` + +## Monitoring + +```typescript +const instances = await env.AI.autorag("_").listInstances(); +console.log(instances.find(i => i.name === "docs")); +``` + +Dashboard shows: files indexed, status, last index time, storage usage. diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-search/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/ai-search/gotchas.md new file mode 100644 index 0000000..04c987f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-search/gotchas.md @@ -0,0 +1,81 @@ +# AI Search Gotchas + +## Type Safety + +**Timestamp precision:** Use seconds (10-digit), not milliseconds. +```typescript +const nowInSeconds = Math.floor(Date.now() / 1000); // Correct +``` + +**Folder prefix matching:** Use `gte` for "starts with" on paths. +```typescript +filters: { column: "folder", operator: "gte", value: "docs/api/" } // Matches nested +``` + +## Filter Limitations + +| Limit | Value | +|-------|-------| +| Max nesting depth | 2 levels | +| Filters per compound | 10 | +| `or` operator | Same column, `eq` only | + +**OR restriction example:** +```typescript +// ✅ Valid: same column, eq only +{ operator: "or", filters: [ + { column: "folder", operator: "eq", value: "docs/" }, + { column: "folder", operator: "eq", value: "guides/" } +]} +``` + +## Indexing Issues + +| Problem | Cause | Solution | +|---------|-------|----------| +| File not indexed | Unsupported format or >4MB | Check format (.md/.txt/.html/.pdf/.doc/.csv/.json) | +| Index out of sync | 6-hour index cycle | Wait or use "Force Sync" (30s rate limit) | +| Empty results | Index incomplete | Check dashboard for indexing status | + +## Auth Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| `AutoRAGUnauthorizedError` | Invalid/missing token | Create Service API token with AI Search permissions | +| `AutoRAGNotFoundError` | Wrong instance name | Verify exact name from dashboard | + +## Performance + +**Slow responses (>3s):** +```typescript +// Add score threshold + limit results +ranking_options: { score_threshold: 0.5 }, +max_num_results: 10 +``` + +**Empty results debug:** +1. Remove filters, test basic query +2. Lower `score_threshold` to 0.1 +3. Check index is populated + +## Limits + +| Resource | Limit | +|----------|-------| +| Instances per account | 10 | +| Files per instance | 100,000 | +| Max file size | 4 MB | +| Index frequency | 6 hours | + +## Anti-Patterns + +**Use env vars for instance names:** +```typescript +const answer = await env.AI.autorag(env.AI_SEARCH_INSTANCE).aiSearch({...}); +``` + +**Handle specific error types:** +```typescript +if (error instanceof AutoRAGNotFoundError) { /* 404 */ } +if (error instanceof AutoRAGUnauthorizedError) { /* 401 */ } +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/ai-search/patterns.md b/plugins/cloudflare/skills/cloudflare/references/ai-search/patterns.md new file mode 100644 index 0000000..4a70f08 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ai-search/patterns.md @@ -0,0 +1,85 @@ +# AI Search Patterns + +## search() vs aiSearch() + +| Use | Method | Returns | +|-----|--------|---------| +| Custom UI, analytics | `search()` | Raw chunks only (~100-300ms) | +| Chatbots, Q&A | `aiSearch()` | AI response + chunks (~500-2000ms) | + +## rewrite_query + +| Setting | Use When | +|---------|----------| +| `true` | User input (typos, vague queries) | +| `false` | LLM-generated queries (already optimized) | + +## Multitenancy (Folder-Based) + +```typescript +const answer = await env.AI.autorag("saas-docs").aiSearch({ + query: "refund policy", + model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + filters: { + column: "folder", + operator: "gte", // "starts with" pattern + value: `tenants/${tenantId}/` + } +}); +``` + +## Streaming + +```typescript +const stream = await env.AI.autorag("docs").aiSearch({ + query, model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast", stream: true +}); +return new Response(stream, { headers: { "Content-Type": "text/event-stream" } }); +``` + +## Score Threshold + +| Threshold | Use | +|-----------|-----| +| 0.3 (default) | Broad recall, exploratory | +| 0.5 | Balanced, production default | +| 0.7 | High precision, critical accuracy | + +## System Prompt Template + +```typescript +const systemPrompt = `You are a documentation assistant. +- Answer ONLY based on provided context +- If context doesn't contain answer, say "I don't have information" +- Include code examples from context`; +``` + +## Compound Filters + +```typescript +// OR: Multiple folders +filters: { + operator: "or", + filters: [ + { column: "folder", operator: "gte", value: "docs/api/" }, + { column: "folder", operator: "gte", value: "docs/auth/" } + ] +} + +// AND: Folder + date +filters: { + operator: "and", + filters: [ + { column: "folder", operator: "gte", value: "docs/" }, + { column: "timestamp", operator: "gte", value: oneWeekAgoSeconds } + ] +} +``` + +## Reranking + +Enable for high-stakes use cases (adds ~300ms latency): + +```typescript +reranking: { enabled: true, model: "@cf/baai/bge-reranker-base" } +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/analytics-engine/README.md b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/README.md new file mode 100644 index 0000000..524123b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/README.md @@ -0,0 +1,92 @@ +# Cloudflare Workers Analytics Engine Reference + +Expert guidance for implementing unlimited-cardinality analytics at scale using Cloudflare Workers Analytics Engine. + +## What is Analytics Engine? + +Time-series analytics database designed for high-cardinality data (millions of unique dimensions). Write data points from Workers, query via SQL API. Use for: +- Custom user-facing analytics dashboards +- Usage-based billing & metering +- Per-customer/per-feature monitoring +- High-frequency instrumentation without performance impact + +**Key Capability:** Track metrics with unlimited unique values (e.g., millions of user IDs, API keys) without performance degradation. + +## Core Concepts + +| Concept | Description | Example | +|---------|-------------|---------| +| **Dataset** | Logical table for related metrics | `api_requests`, `user_events` | +| **Data Point** | Single measurement with timestamp | One API request's metrics | +| **Blobs** | String dimensions (max 20) | endpoint, method, status, user_id | +| **Doubles** | Numeric values (max 20) | latency_ms, request_count, bytes | +| **Indexes** | Filtered blobs for efficient queries | customer_id, api_key | + +## Reading Order + +| Task | Start Here | Then Read | +|------|------------|-----------| +| **First-time setup** | [configuration.md](configuration.md) → [api.md](api.md) → [patterns.md](patterns.md) | | +| **Writing data** | [api.md](api.md) → [gotchas.md](gotchas.md) (sampling) | | +| **Querying data** | [api.md](api.md) (SQL API) → [patterns.md](patterns.md) (examples) | | +| **Debugging** | [gotchas.md](gotchas.md) → [api.md](api.md) (limits) | | +| **Optimization** | [patterns.md](patterns.md) (anti-patterns) → [gotchas.md](gotchas.md) | | + +## When to Use Analytics Engine + +``` +Need to track metrics? → Yes + ↓ +Millions of unique dimension values? → Yes + ↓ + Need real-time queries? → Yes + ↓ + Use Analytics Engine ✓ + +Alternative scenarios: +- Low cardinality (<10k unique values) → Workers Analytics (free tier) +- Complex joins/relations → D1 Database +- Logs/debugging → Tail Workers (logpush) +- External tools → Send to external analytics (Datadog, etc.) +``` + +## Quick Start + +1. Add binding to `wrangler.jsonc`: +```jsonc +{ + "analytics_engine_datasets": [ + { "binding": "ANALYTICS", "dataset": "my_events" } + ] +} +``` + +2. Write data points (fire-and-forget, no await): +```typescript +env.ANALYTICS.writeDataPoint({ + blobs: ["/api/users", "GET", "200"], + doubles: [145.2, 1], // latency_ms, count + indexes: [customerId] +}); +``` + +3. Query via SQL API (HTTP): +```sql +SELECT blob1, SUM(double2) AS total_requests +FROM my_events +WHERE index1 = 'customer_123' + AND timestamp >= NOW() - INTERVAL '7' DAY +GROUP BY blob1 +ORDER BY total_requests DESC +``` + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, bindings, TypeScript types, limits +- **[api.md](api.md)** - `writeDataPoint()`, SQL API, query syntax +- **[patterns.md](patterns.md)** - Use cases, examples, anti-patterns +- **[gotchas.md](gotchas.md)** - Sampling, index selection, troubleshooting + +## See Also + +- [Cloudflare Analytics Engine Docs](https://developers.cloudflare.com/analytics/analytics-engine/) diff --git a/plugins/cloudflare/skills/cloudflare/references/analytics-engine/api.md b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/api.md new file mode 100644 index 0000000..20e7084 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/api.md @@ -0,0 +1,112 @@ +# Analytics Engine API Reference + +## Writing Data + +### `writeDataPoint()` + +Fire-and-forget (returns `void`, not Promise). Writes happen asynchronously. + +```typescript +interface AnalyticsEngineDataPoint { + blobs?: string[]; // Up to 20 strings (dimensions), 16KB each + doubles?: number[]; // Up to 20 numbers (metrics) + indexes?: string[]; // 1 indexed string for high-cardinality filtering +} + +env.ANALYTICS.writeDataPoint({ + blobs: ["/api/users", "GET", "200"], + doubles: [145.2, 1], // latency_ms, count + indexes: ["customer_abc123"] +}); +``` + +**Behaviors:** No await needed, no error thrown (check tail logs), auto-sampled at high volumes, auto-timestamped. + +**Blob vs Index:** Blob for GROUP BY (<100k unique), Index for filter-only (millions unique). + +### Full Example + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const start = Date.now(); + const url = new URL(request.url); + try { + const response = await handleRequest(request); + env.ANALYTICS.writeDataPoint({ + blobs: [url.pathname, request.method, response.status.toString()], + doubles: [Date.now() - start, 1], + indexes: [request.headers.get("x-api-key") || "anonymous"] + }); + return response; + } catch (error) { + env.ANALYTICS.writeDataPoint({ + blobs: [url.pathname, request.method, "500"], + doubles: [Date.now() - start, 1, 0], + }); + throw error; + } + } +}; +``` + +## SQL API (External Only) + +```bash +curl -X POST https://api.cloudflare.com/client/v4/accounts/{account_id}/analytics_engine/sql \ + -H "Authorization: Bearer $TOKEN" \ + -d "SELECT blob1 AS endpoint, COUNT(*) AS requests FROM dataset WHERE timestamp >= NOW() - INTERVAL '1' HOUR GROUP BY blob1" +``` + +### Column References + +```sql +-- blob1..blob20, double1..double20, index1, timestamp +SELECT blob1 AS endpoint, SUM(double1) AS latency, COUNT(*) AS requests +FROM my_dataset +WHERE index1 = 'customer_123' AND timestamp >= NOW() - INTERVAL '7' DAY +GROUP BY blob1 +HAVING COUNT(*) > 100 +ORDER BY requests DESC LIMIT 100 +``` + +**Aggregations:** `SUM()`, `AVG()`, `COUNT()`, `MIN()`, `MAX()`, `quantile(0.95)()` + +**Time ranges:** `NOW() - INTERVAL '1' HOUR`, `BETWEEN '2026-01-01' AND '2026-01-31'` + +### Query Examples + +```sql +-- Top endpoints +SELECT blob1, COUNT(*) AS requests, AVG(double1) AS avg_latency +FROM api_requests WHERE timestamp >= NOW() - INTERVAL '24' HOUR +GROUP BY blob1 ORDER BY requests DESC LIMIT 20 + +-- Error rate +SELECT blob1, COUNT(*) AS total, + SUM(CASE WHEN blob3 LIKE '5%' THEN 1 ELSE 0 END) AS errors +FROM api_requests WHERE timestamp >= NOW() - INTERVAL '1' HOUR +GROUP BY blob1 HAVING total > 50 + +-- P95 latency +SELECT blob1, quantile(0.95)(double1) AS p95 +FROM api_requests GROUP BY blob1 +``` + +## Response Format + +```json +{"data": [{"endpoint": "/api/users", "requests": 1523}], "rows": 2} +``` + +## Limits + +| Resource | Limit | +|----------|-------| +| Blobs/Doubles per point | 20 each | +| Indexes per point | 1 | +| Blob/Index size | 16KB | +| Data retention | 90 days | +| Query timeout | 30s | + +**Critical:** High write volumes (>1M/min) trigger automatic sampling. diff --git a/plugins/cloudflare/skills/cloudflare/references/analytics-engine/configuration.md b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/configuration.md new file mode 100644 index 0000000..a71af96 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/configuration.md @@ -0,0 +1,112 @@ +# Analytics Engine Configuration + +## Setup + +1. Add binding to `wrangler.jsonc` +2. Deploy Worker +3. Dataset created automatically on first write +4. Query via SQL API + +## wrangler.jsonc + +```jsonc +{ + "name": "my-worker", + "analytics_engine_datasets": [ + { "binding": "ANALYTICS", "dataset": "my_events" } + ] +} +``` + +Multiple datasets for separate concerns: +```jsonc +{ + "analytics_engine_datasets": [ + { "binding": "API_ANALYTICS", "dataset": "api_requests" }, + { "binding": "USER_EVENTS", "dataset": "user_activity" } + ] +} +``` + +## TypeScript + +```typescript +interface Env { + ANALYTICS: AnalyticsEngineDataset; +} + +export default { + async fetch(request: Request, env: Env) { + // No await - returns void, fire-and-forget + env.ANALYTICS.writeDataPoint({ + blobs: [pathname, method, status], // String dimensions (max 20) + doubles: [latency, 1], // Numeric metrics (max 20) + indexes: [apiKey] // High-cardinality filter (max 1) + }); + return response; + } +}; +``` + +## Data Point Limits + +| Field | Limit | SQL Access | +|-------|-------|------------| +| blobs | 20 strings, 16KB each | `blob1`...`blob20` | +| doubles | 20 numbers | `double1`...`double20` | +| indexes | 1 string, 16KB | `index1` | + +## Write Behavior + +| Scenario | Behavior | +|----------|----------| +| <1M writes/min | All accepted | +| >1M writes/min | Automatic sampling | +| Invalid data | Silent failure (check tail logs) | + +**Mitigate sampling:** Pre-aggregate, use multiple datasets, write only critical metrics. + +## Query Limits + +| Resource | Limit | +|----------|-------| +| Query timeout | 30 seconds | +| Data retention | 90 days (default) | +| Result size | ~10MB | + +## Cost + +**Free tier:** 10M writes/month, 1M reads/month + +**Paid:** $0.05 per 1M writes, $1.00 per 1M reads + +## Environment-Specific + +```jsonc +{ + "analytics_engine_datasets": [ + { "binding": "ANALYTICS", "dataset": "prod_events" } + ], + "env": { + "staging": { + "analytics_engine_datasets": [ + { "binding": "ANALYTICS", "dataset": "staging_events" } + ] + } + } +} +``` + +## Monitoring + +```bash +npx wrangler tail # Check for sampling/write errors +``` + +```sql +-- Check write activity +SELECT DATE_TRUNC('hour', timestamp) AS hour, COUNT(*) AS writes +FROM my_dataset +WHERE timestamp >= NOW() - INTERVAL '24' HOUR +GROUP BY hour +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/analytics-engine/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/gotchas.md new file mode 100644 index 0000000..ed1767f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/gotchas.md @@ -0,0 +1,85 @@ +# Analytics Engine Gotchas + +## Critical Issues + +### Sampling at High Volumes + +**Problem:** Queries return fewer points than written at >1M writes/min. + +**Solution:** +```typescript +// Pre-aggregate before writing +let buffer = { count: 0, total: 0 }; +buffer.count++; buffer.total += value; + +// Write once per second instead of per request +if (Date.now() % 1000 === 0) { + env.ANALYTICS.writeDataPoint({ doubles: [buffer.count, buffer.total] }); +} +``` + +**Detection:** `npx wrangler tail` → look for "sampling enabled" + +### writeDataPoint Returns void + +```typescript +// ❌ Pointless await +await env.ANALYTICS.writeDataPoint({...}); + +// ✅ Fire-and-forget +env.ANALYTICS.writeDataPoint({...}); +``` + +Writes can fail silently. Check tail logs. + +### Index vs Blob + +| Cardinality | Use | Example | +|-------------|-----|---------| +| Millions | **Index** | user_id, api_key | +| Hundreds | **Blob** | endpoint, status_code, country | + +```typescript +// ✅ Correct +{ blobs: [method, path, status], indexes: [userId] } +``` + +### Can't Query from Workers + +Query API requires HTTP auth. Use external service or cache in KV/D1. + +### No Custom Timestamps + +Auto-generated at write time. Store original in blob if needed. + +## Common Errors + +| Error | Fix | +|-------|-----| +| Binding not found | Check wrangler.jsonc, redeploy | +| No data in query | Wait 30s; check dataset name; check time range | +| Query timeout | Add time filter; use index for filtering | + +## Limits + +| Resource | Limit | +|----------|-------| +| Blobs per point | 20 | +| Doubles per point | 20 | +| Indexes per point | 1 | +| Blob/Index size | 16KB | +| Write rate (no sampling) | ~1M/min | +| Retention | 90 days | +| Query timeout | 30s | + +## Best Practices + +✅ Pre-aggregate at high volumes +✅ Use index for high-cardinality (millions) +✅ Always include time filter in queries +✅ Design schema before coding + +❌ Don't await writeDataPoint +❌ Don't use index for low-cardinality +❌ Don't query without time range +❌ Don't assume all writes succeed diff --git a/plugins/cloudflare/skills/cloudflare/references/analytics-engine/patterns.md b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/patterns.md new file mode 100644 index 0000000..dc3fc5d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/analytics-engine/patterns.md @@ -0,0 +1,83 @@ +# Analytics Engine Patterns + +## Use Cases + +| Use Case | Key Metrics | Index On | +|----------|-------------|----------| +| API Metering | requests, bytes, compute_units | api_key | +| Feature Usage | feature, action, duration | user_id | +| Error Tracking | error_type, endpoint, count | customer_id | +| Performance | latency_ms, cache_status | endpoint | +| A/B Testing | variant, conversions | user_id | + +## API Metering (Billing) + +```typescript +env.ANALYTICS.writeDataPoint({ + blobs: [pathname, method, status, tier], + doubles: [1, computeUnits, bytes, latencyMs], + indexes: [apiKey] +}); + +// Query: Monthly usage by customer +// SELECT index1 AS api_key, SUM(double2) AS compute_units +// FROM usage WHERE timestamp >= DATE_TRUNC('month', NOW()) GROUP BY index1 +``` + +## Error Tracking + +```typescript +env.ANALYTICS.writeDataPoint({ + blobs: [endpoint, method, errorName, errorMessage.slice(0, 1000)], + doubles: [1, timeToErrorMs], + indexes: [customerId] +}); +``` + +## Performance Monitoring + +```typescript +env.ANALYTICS.writeDataPoint({ + blobs: [pathname, method, cacheStatus, status], + doubles: [latencyMs, 1], + indexes: [userId] +}); + +// Query: P95 latency by endpoint +// SELECT blob1, quantile(0.95)(double1) AS p95_ms FROM perf GROUP BY blob1 +``` + +## Anti-Patterns + +| ❌ Wrong | ✅ Correct | +|----------|-----------| +| `await writeDataPoint()` | `writeDataPoint()` (fire-and-forget) | +| `indexes: [method]` (low cardinality) | `blobs: [method]`, `indexes: [userId]` | +| `blobs: [JSON.stringify(obj)]` | Store ID in blob, full object in D1/KV | +| Write every request at 10M/min | Pre-aggregate per second | +| Query from Worker | Query from external service/API | + +## Best Practices + +1. **Design schema upfront** - Document blob/double/index assignments +2. **Always include count metric** - `doubles: [latency, 1]` for AVG calculations +3. **Use enums for blobs** - Consistent values like `Status.SUCCESS` +4. **Handle sampling** - Use ratios (avg_latency = SUM(latency)/SUM(count)) +5. **Test queries early** - Validate schema before heavy writes + +## Schema Template + +```typescript +/** + * Dataset: my_metrics + * + * Blobs: + * blob1: endpoint, blob2: method, blob3: status + * + * Doubles: + * double1: latency_ms, double2: count (always 1) + * + * Indexes: + * index1: customer_id (high cardinality) + */ +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/api-shield/README.md b/plugins/cloudflare/skills/cloudflare/references/api-shield/README.md new file mode 100644 index 0000000..86613c8 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api-shield/README.md @@ -0,0 +1,44 @@ +# Cloudflare API Shield Reference + +Expert guidance for API Shield - comprehensive API security suite for discovery, protection, and monitoring. + +## Reading Order + +| Task | Files to Read | +|------|---------------| +| Initial setup | README → configuration.md | +| Implement JWT validation | configuration.md → api.md | +| Add schema validation | configuration.md → patterns.md | +| Detect API attacks | patterns.md → api.md | +| Debug issues | gotchas.md | + +## Feature Selection + +What protection do you need? + +``` +├─ Validate request/response structure → Schema Validation 2.0 (configuration.md) +├─ Verify auth tokens → JWT Validation (configuration.md) +├─ Client certificates → mTLS (configuration.md) +├─ Detect BOLA attacks → BOLA Detection (patterns.md) +├─ Track auth coverage → Auth Posture (patterns.md) +├─ Stop volumetric abuse → Abuse Detection (patterns.md) +└─ Discover shadow APIs → API Discovery (api.md) +``` + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, session identifiers, rules, token/mTLS configs +- **[api.md](api.md)** - Endpoint management, discovery, validation APIs, GraphQL operations +- **[patterns.md](patterns.md)** - Common patterns, progressive rollout, OWASP mappings, workflows +- **[gotchas.md](gotchas.md)** - Troubleshooting, false positives, performance, best practices + +## Quick Start + +API Shield: Enterprise-grade API security (Discovery, Schema Validation 2.0, JWT, mTLS, BOLA Detection, Auth Posture). Available as Enterprise add-on with preview access. + +## See Also + +- [API Shield Docs](https://developers.cloudflare.com/api-shield/) +- [API Reference](https://developers.cloudflare.com/api/resources/api_gateway/) +- [OWASP API Security Top 10](https://owasp.org/www-project-api-security/) diff --git a/plugins/cloudflare/skills/cloudflare/references/api-shield/api.md b/plugins/cloudflare/skills/cloudflare/references/api-shield/api.md new file mode 100644 index 0000000..c833501 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api-shield/api.md @@ -0,0 +1,141 @@ +# API Reference + +Base: `/zones/{zone_id}/api_gateway` + +## Endpoints + +```bash +GET /operations # List +GET /operations/{op_id} # Get single +POST /operations/item # Create: {endpoint,host,method} +POST /operations # Bulk: {operations:[{endpoint,host,method}]} +DELETE /operations/{op_id} # Delete +DELETE /operations # Bulk delete: {operation_ids:[...]} +``` + +## Discovery + +```bash +GET /discovery/operations # List discovered +PATCH /discovery/operations/{op_id} # Update: {state:"saved"|"ignored"} +PATCH /discovery/operations # Bulk: {operation_ids:{id:{state}}} +GET /discovery # OpenAPI export +``` + +## Config + +```bash +GET /configuration # Get session ID config +PUT /configuration # Update: {auth_id_characteristics:[{name,type:"header"|"cookie"}]} +``` + +## Token Validation + +```bash +GET /token_validation # List +POST /token_validation # Create: {name,location:{header:"..."},jwks:"..."} +POST /jwt_validation_rules # Rule: {name,hostname,token_validation_id,action:"block"} +``` + +## Workers Integration + +### Access JWT Claims +```js +export default { + async fetch(req, env) { + // Access validated JWT payload + const jwt = req.cf?.jwt?.payload?.[env.JWT_CONFIG_ID]?.[0]; + if (jwt) { + const userId = jwt.sub; + const role = jwt.role; + } + } +} +``` + +### Access mTLS Info +```js +export default { + async fetch(req, env) { + const tls = req.cf?.tlsClientAuth; + if (tls?.certVerified === 'SUCCESS') { + const fingerprint = tls.certFingerprintSHA256; + // Authenticated client + } + } +} +``` + +### Dynamic JWKS Update +```js +export default { + async scheduled(event, env) { + const jwks = await (await fetch('https://auth.example.com/.well-known/jwks.json')).json(); + await fetch(`https://api.cloudflare.com/client/v4/zones/${env.ZONE_ID}/api_gateway/token_validation/${env.CONFIG_ID}`, { + method: 'PATCH', + headers: {'Authorization': `Bearer ${env.CF_API_TOKEN}`, 'Content-Type': 'application/json'}, + body: JSON.stringify({jwks: JSON.stringify(jwks)}) + }); + } +} +``` + +## Firewall Fields + +### Core Fields +```js +cf.api_gateway.auth_id_present // Session ID present +cf.api_gateway.request_violates_schema // Schema violation +cf.api_gateway.fallthrough_triggered // No endpoint match +cf.tls_client_auth.cert_verified // mTLS cert valid +cf.tls_client_auth.cert_fingerprint_sha256 +``` + +### JWT Validation (2026) +```js +// Modern validation syntax +is_jwt_valid(http.request.jwt.payload["{config_id}"][0]) + +// Legacy (still supported) +cf.api_gateway.jwt_claims_valid + +// Extract claims +lookup_json_string(http.request.jwt.payload["{config_id}"][0], "claim_name") +``` + +### Risk Labels (2026) +```js +// BOLA detection +cf.api_gateway.cf-risk-bola-enumeration // Sequential resource access detected +cf.api_gateway.cf-risk-bola-pollution // Parameter pollution detected + +// Authentication posture +cf.api_gateway.cf-risk-missing-auth // Endpoint lacks authentication +cf.api_gateway.cf-risk-mixed-auth // Inconsistent auth patterns +``` + +## BOLA Detection + +```bash +GET /user_schemas/{schema_id}/bola # Get BOLA config +PATCH /user_schemas/{schema_id}/bola # Update: {enabled:true} +``` + +## Auth Posture + +```bash +GET /discovery/authentication_posture # List unprotected endpoints +``` + +## GraphQL Protection + +```bash +GET /settings/graphql_protection # Get limits +PUT /settings/graphql_protection # Set: {max_depth,max_size} +``` + +## See Also + +- [configuration.md](configuration.md) - Setup guides for all features +- [patterns.md](patterns.md) - Firewall rules and common patterns +- [API Gateway API Docs](https://developers.cloudflare.com/api/resources/api_gateway/) diff --git a/plugins/cloudflare/skills/cloudflare/references/api-shield/configuration.md b/plugins/cloudflare/skills/cloudflare/references/api-shield/configuration.md new file mode 100644 index 0000000..f95744c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api-shield/configuration.md @@ -0,0 +1,192 @@ +# Configuration + +## Schema Validation 2.0 Setup + +> ⚠️ **Classic Schema Validation deprecated.** Use Schema Validation 2.0. + +**Upload schema (Dashboard):** +``` +Security > API Shield > Schema Validation > Add validation +- Upload .yml/.yaml/.json (OpenAPI v3.0) +- Endpoints auto-added to Endpoint Management +- Action: Log | Block | None +- Body inspection: JSON payloads +``` + +**Change validation action:** +``` +Security > API Shield > Settings > Schema Validation +Per-endpoint: Filter → ellipses → Change action +Default action: Set global mitigation action +``` + +**Migration from Classic:** +``` +1. Export existing schema (if available) +2. Delete all Classic schema validation rules +3. Wait 5 min for cache clear +4. Re-upload via Schema Validation 2.0 interface +5. Verify in Security > Events +``` + +**Fallthrough rule** (catch-all unknown endpoints): +``` +Security > API Shield > Settings > Fallthrough > Use Template +- Select hostnames +- Create rule with cf.api_gateway.fallthrough_triggered +- Action: Log (discover) or Block (strict) +``` + +**Body inspection:** Supports `application/json`, `*/*`, `application/*`. Disable origin MIME sniffing to prevent bypasses. + +## JWT Validation + +**Setup token config:** +``` +Security > API Shield > Settings > JWT Settings > Add configuration +- Name: "Auth0 JWT Config" +- Location: Header/Cookie + name (e.g., "Authorization") +- JWKS: Paste public keys from IdP +``` + +**Create validation rule:** +``` +Security > API Shield > API Rules > Add rule +- Hostname: api.example.com +- Deselect endpoints to ignore +- Token config: Select config +- Enforce presence: Ignore or Mark as non-compliant +- Action: Log/Block/Challenge +``` + +**Rate limit by JWT claim:** +```wirefilter +lookup_json_string(http.request.jwt.claims["{config_id}"][0], "sub") +``` + +**Special cases:** +- Two JWTs, different IdPs: Create 2 configs, select both, "Validate all" +- IdP migration: 2 configs + 2 rules, adjust actions per state +- Bearer prefix: API Shield handles with/without +- Nested claims: Dot notation `user.email` + +## Mutual TLS (mTLS) + +**Setup:** +``` +SSL/TLS > Client Certificates > Create Certificate +- Generate CF-managed CA (all plans) +- Upload custom CA (Enterprise, max 5) +``` + +**Configure mTLS rule:** +``` +Security > API Shield > mTLS +- Select hostname(s) +- Choose certificate(s) +- Action: Block/Log/Challenge +``` + +**Test:** +```bash +openssl req -x509 -newkey rsa:4096 -keyout client-key.pem -out client-cert.pem -days 365 +curl https://api.example.com/endpoint --cert client-cert.pem --key client-key.pem +``` + +## Session Identifiers + +Critical for BOLA Detection, Sequence Mitigation, and analytics. Configure header/cookie that uniquely IDs API users. + +**Examples:** JWT sub claim, session token, API key, custom user ID header + +**Configure:** +``` +Security > API Shield > Settings > Session Identifiers +- Type: Header/Cookie +- Name: "X-User-ID" or "Authorization" +``` + +## BOLA Detection + +Detects Broken Object Level Authorization attacks (enumeration + parameter pollution). + +**Enable:** +``` +Security > API Shield > Schema Validation > [Select Schema] > BOLA Detection +- Enable detection +- Threshold: Sensitivity level (Low/Medium/High) +- Action: Log or Block +``` + +**Requirements:** +- Schema Validation 2.0 enabled +- Session identifiers configured +- Minimum traffic: 1000+ requests/day per endpoint + +## Authentication Posture + +Identifies unprotected or inconsistently protected endpoints. + +**View report:** +``` +Security > API Shield > Authentication Posture +- Shows endpoints lacking JWT/mTLS +- Highlights mixed authentication patterns +``` + +**Remediate:** +1. Review flagged endpoints +2. Add JWT validation rules +3. Configure mTLS for sensitive endpoints +4. Monitor posture score + +## Volumetric Abuse + GraphQL + +**Volumetric Abuse Detection:** +`Security > API Shield > Settings > Volumetric Abuse Detection` +- Enable per-endpoint monitoring, set thresholds, action: Log | Challenge | Block + +**GraphQL Protection:** +`Security > API Shield > Settings > GraphQL Protection` +- Max query depth: 10, max size: 100KB, block introspection (production) + +## Terraform + +```hcl +# Session identifier +resource "cloudflare_api_shield" "main" { + zone_id = var.zone_id + auth_id_characteristics { + type = "header" + name = "Authorization" + } +} + +# Add endpoint +resource "cloudflare_api_shield_operation" "users_get" { + zone_id = var.zone_id + method = "GET" + host = "api.example.com" + endpoint = "/api/users/{id}" +} + +# JWT validation rule +resource "cloudflare_ruleset" "jwt_validation" { + zone_id = var.zone_id + name = "API JWT Validation" + kind = "zone" + phase = "http_request_firewall_custom" + + rules { + action = "block" + expression = "(http.host eq \"api.example.com\" and not is_jwt_valid(http.request.jwt.payload[\"{config_id}\"][0]))" + description = "Block invalid JWTs" + } +} +``` + +## See Also + +- [api.md](api.md) - API endpoints and Workers integration +- [patterns.md](patterns.md) - Firewall rules and deployment patterns +- [gotchas.md](gotchas.md) - Troubleshooting and limits diff --git a/plugins/cloudflare/skills/cloudflare/references/api-shield/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/api-shield/gotchas.md new file mode 100644 index 0000000..255517f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api-shield/gotchas.md @@ -0,0 +1,125 @@ +# Gotchas & Troubleshooting + +## Common Errors + +### "Schema Validation 2.0 not working after migration" + +**Cause:** Classic rules still active, conflicting with new system +**Solution:** +1. Delete ALL Classic schema validation rules +2. Clear Cloudflare cache (wait 5 min) +3. Re-upload schema via new Schema Validation 2.0 interface +4. Verify in Security > Events +5. Check action is set (Log/Block) + +### "Schema validation blocking valid requests" + +**Cause:** Schema too restrictive, missing fields, or incorrect types +**Solution:** +1. Check Firewall Events for violation details +2. Review schema in Settings +3. Test schema in Swagger Editor +4. Use Log mode to validate before blocking +5. Update schema with correct specifications +6. Ensure Schema Validation 2.0 (not Classic) + +### "JWT validation failing" + +**Cause:** JWKS mismatch with IdP, expired token, wrong header/cookie name, or clock skew +**Solution:** +1. Verify JWKS matches IdP configuration +2. Check token `exp` claim is valid +3. Confirm header/cookie name matches config +4. Test token at jwt.io +5. Account for clock skew (±5 min tolerance) +6. Use modern syntax: `is_jwt_valid(http.request.jwt.payload["{config_id}"][0])` + +### "BOLA detection false positives" + +**Cause:** Legitimate sequential access patterns, bulk operations, or sensitivity too high +**Solution:** +1. Review BOLA events in Security > Events +2. Lower sensitivity threshold (High → Medium → Low) +3. Exclude legitimate bulk operations from detection +4. Ensure session identifiers uniquely identify users +5. Verify minimum traffic requirements met (1000+ req/day) + +### "Risk labels not appearing in firewall rules" + +**Cause:** Feature not enabled, insufficient traffic, or missing session identifiers +**Solution:** +1. Verify Schema Validation 2.0 enabled +2. Enable BOLA Detection in schema settings +3. Configure session identifiers (required for BOLA) +4. Wait 24-48h for ML model training +5. Check minimum traffic thresholds met + +### "Endpoint discovery not finding APIs" + +**Cause:** Insufficient traffic (<500 reqs/10d), non-2xx responses, Worker direct requests, or incorrect session ID config +**Solution:** Ensure 500+ requests in 10 days, 2xx responses from edge (not Workers direct), configure session IDs correctly. ML updates daily. + +### "Sequence detection false positives" + +**Cause:** Lookback window issues, non-unique session IDs, or model sensitivity +**Solution:** +1. Review lookback settings (10 reqs to managed endpoints, 10min window) +2. Ensure session ID uniqueness per user (not shared tokens) +3. Adjust positive/negative model balance +4. Exclude legitimate workflows from detection + +### "GraphQL protection blocking valid queries" + +**Cause:** Query depth/size limits too restrictive, complex but legitimate queries +**Solution:** +1. Review blocked query patterns in Security > Events +2. Increase max_depth (default: 10) if needed +3. Increase max_size (default: 100KB) for complex queries +4. Whitelist specific query signatures +5. Use Log mode to tune before blocking + +### "Token invalid" + +**Cause:** Configuration error, JWKS mismatch, or expired token +**Solution:** Verify config matches IdP, update JWKS, check token expiration + +### "Schema violation" + +**Cause:** Missing required fields, wrong data types, or spec mismatch +**Solution:** Review schema against actual requests, ensure all required fields present, validate types match spec + +### "Fallthrough" + +**Cause:** Unknown endpoint or pattern mismatch +**Solution:** Update schema with all endpoints, check path pattern matching + +### "mTLS failed" + +**Cause:** Certificate untrusted/expired or wrong CA +**Solution:** Verify cert chain, check expiration, confirm correct CA uploaded + +## Limits (2026) + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| OpenAPI version | v3.0.x only | No external refs, must be valid | +| Schema operations | 10K (Enterprise) | Contact for higher limits | +| JWT validation sources | Headers/cookies only | No query params/body | +| Endpoint discovery | 500+ reqs/10d | Minimum for ML model | +| Path normalization | Automatic | `/profile/238` → `/profile/{var1}` | +| Schema parameters | No `content` field | No object param validation | +| BOLA detection | 1000+ reqs/day/endpoint | Per-endpoint minimum | +| Session ID uniqueness | Required | BOLA/Sequence need unique IDs | +| GraphQL max depth | 1-50 | Default: 10 | +| GraphQL max size | 1KB-1MB | Default: 100KB | +| JWT claim nesting | 10 levels max | Use dot notation | +| mTLS CA certificates | 5 custom max | CF-managed unlimited | +| Schema upload size | 5MB max | Compressed OpenAPI spec | +| Volumetric abuse baseline | 7 days training | Initial ML period | +| Auth Posture refresh | Daily | Updated nightly | + +## See Also + +- [configuration.md](configuration.md) - Setup guides to avoid common issues +- [patterns.md](patterns.md) - Best practices and progressive rollout +- [API Shield Docs](https://developers.cloudflare.com/api-shield/) diff --git a/plugins/cloudflare/skills/cloudflare/references/api-shield/patterns.md b/plugins/cloudflare/skills/cloudflare/references/api-shield/patterns.md new file mode 100644 index 0000000..5721dba --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api-shield/patterns.md @@ -0,0 +1,180 @@ +# Patterns & Use Cases + +## Protect API with Schema + JWT + +```bash +# 1. Upload OpenAPI schema +POST /zones/{zone_id}/api_gateway/user_schemas + +# 2. Configure JWT validation +POST /zones/{zone_id}/api_gateway/token_validation +{ + "name": "Auth0", + "location": {"header": "Authorization"}, + "jwks": "{...}" +} + +# 3. Create JWT rule +POST /zones/{zone_id}/api_gateway/jwt_validation_rules + +# 4. Set schema validation action +PUT /zones/{zone_id}/api_gateway/settings/schema_validation +{"validation_default_mitigation_action": "block"} +``` + +## Progressive Rollout + +``` +1. Log mode: Observe false positives + - Schema: Action = Log + - JWT: Action = Log + +2. Block subset: Protect critical endpoints + - Change specific endpoint actions to Block + - Monitor firewall events + +3. Full enforcement: Block all violations + - Change default action to Block + - Handle fallthrough with custom rule +``` + +## BOLA Detection + +### Enumeration Detection +Detects sequential resource access (e.g., `/users/1`, `/users/2`, `/users/3`). + +```javascript +// Block BOLA enumeration attempts +(cf.api_gateway.cf-risk-bola-enumeration and http.host eq "api.example.com") +// Action: Block or Challenge +``` + +### Parameter Pollution +Detects duplicate/excessive parameters in requests. + +```javascript +// Block parameter pollution +(cf.api_gateway.cf-risk-bola-pollution and http.host eq "api.example.com") +// Action: Block +``` + +### Combined BOLA Protection +```javascript +// Comprehensive BOLA rule +(cf.api_gateway.cf-risk-bola-enumeration or cf.api_gateway.cf-risk-bola-pollution) +and http.host eq "api.example.com" +// Action: Block +``` + +## Authentication Posture + +### Detect Missing Auth +```javascript +// Log endpoints lacking authentication +(cf.api_gateway.cf-risk-missing-auth and http.host eq "api.example.com") +// Action: Log (for audit) +``` + +### Detect Mixed Auth +```javascript +// Alert on inconsistent auth patterns +(cf.api_gateway.cf-risk-mixed-auth and http.host eq "api.example.com") +// Action: Log (review required) +``` + +## Fallthrough Detection (Shadow APIs) + +```javascript +// WAF Custom Rule +(cf.api_gateway.fallthrough_triggered and http.host eq "api.example.com") +// Action: Log (discover unknown) or Block (strict) +``` + +## Rate Limiting by User + +```javascript +// Rate Limiting Rule (modern syntax) +(http.host eq "api.example.com" and + is_jwt_valid(http.request.jwt.payload["{config_id}"][0])) + +// Rate: 100 req/60s +// Counting expression: lookup_json_string(http.request.jwt.payload["{config_id}"][0], "sub") +``` + +## Volumetric Abuse Response + +```javascript +// Detect abnormal traffic spikes +(cf.api_gateway.volumetric_abuse_detected and http.host eq "api.example.com") +// Action: Challenge or Rate Limit + +// Combined with rate limiting +(cf.api_gateway.volumetric_abuse_detected or + cf.threat_score gt 50) and http.host eq "api.example.com" +// Action: JS Challenge +``` + +## GraphQL Protection + +```javascript +// Block oversized queries +(http.request.uri.path eq "/graphql" and + cf.api_gateway.graphql_query_size gt 100000) +// Action: Block + +// Block deep nested queries +(http.request.uri.path eq "/graphql" and + cf.api_gateway.graphql_query_depth gt 10) +// Action: Block +``` + +## Architecture Patterns + +**Public API:** Discovery + Schema Validation 2.0 + JWT + Rate Limiting + Bot Management +**Partner API:** mTLS + Schema Validation + Sequence Mitigation +**Internal API:** Discovery + Schema Learning + Auth Posture + +## OWASP API Security Top 10 Mapping (2026) + +| OWASP Issue | API Shield Solutions | +|-------------|---------------------| +| API1:2023 Broken Object Level Authorization | **BOLA Detection** (enumeration + pollution), Sequence mitigation, Schema, JWT, Rate Limiting | +| API2:2023 Broken Authentication | **Auth Posture**, mTLS, JWT validation, Bot Management | +| API3:2023 Broken Object Property Auth | Schema validation, JWT validation | +| API4:2023 Unrestricted Resource Access | Rate Limiting, **Volumetric Abuse Detection**, **GraphQL Protection**, Bot Management | +| API5:2023 Broken Function Level Auth | Schema validation, JWT validation, Auth Posture | +| API6:2023 Unrestricted Business Flows | Sequence mitigation, Bot Management | +| API7:2023 SSRF | Schema validation, WAF managed rules | +| API8:2023 Security Misconfiguration | **Schema Validation 2.0**, Auth Posture, WAF rules | +| API9:2023 Improper Inventory Management | **API Discovery**, Schema learning, Auth Posture | +| API10:2023 Unsafe API Consumption | JWT validation, Schema validation, WAF managed | + +## Monitoring + +**Security Events:** `Security > Events` → Filter: Action = block, Service = API Shield +**Firewall Analytics:** `Analytics > Security` → Filter by `cf.api_gateway.*` fields +**Logpush fields:** APIGatewayAuthIDPresent, APIGatewayRequestViolatesSchema, APIGatewayFallthroughDetected, JWTValidationResult + +## Availability (2026) + +| Feature | Availability | Notes | +|---------|-------------|-------| +| mTLS (CF-managed CA) | All plans | Self-service | +| Endpoint Management | All plans | Limited operations | +| Schema Validation 2.0 | All plans | Limited operations | +| API Discovery | Enterprise | 10K+ ops | +| JWT Validation | Enterprise add-on | Full validation | +| BOLA Detection | Enterprise add-on | Requires session IDs | +| Auth Posture | Enterprise add-on | Security audit | +| Volumetric Abuse Detection | Enterprise add-on | Traffic analysis | +| GraphQL Protection | Enterprise add-on | Query limits | +| Sequence Mitigation | Enterprise (beta) | Contact team | +| Full Suite | Enterprise add-on | All features | + +**Enterprise limits:** 10K operations (contact for higher). Preview access available for non-contract evaluation. + +## See Also + +- [configuration.md](configuration.md) - Setup all features before creating rules +- [api.md](api.md) - Firewall field reference and API endpoints +- [gotchas.md](gotchas.md) - Common issues and limits diff --git a/plugins/cloudflare/skills/cloudflare/references/api/README.md b/plugins/cloudflare/skills/cloudflare/references/api/README.md new file mode 100644 index 0000000..5e02109 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api/README.md @@ -0,0 +1,65 @@ +# Cloudflare API Integration + +Guide for working with Cloudflare's REST API - authentication, SDK usage, common patterns, and troubleshooting. + +## Quick Decision Tree + +``` +How are you calling the Cloudflare API? +├─ From Workers runtime → Use bindings, not REST API (see ../bindings/) +├─ Server-side (Node/Python/Go) → Official SDK (see api.md) +├─ CLI/scripts → Wrangler or curl (see configuration.md) +├─ Infrastructure-as-code → See ../pulumi/ or ../terraform/ +└─ One-off requests → curl examples (see api.md) +``` + +## SDK Selection + +| Language | Package | Best For | Default Retries | +|----------|---------|----------|-----------------| +| TypeScript | `cloudflare` | Node.js, Bun, Next.js, Workers | 2 | +| Python | `cloudflare` | FastAPI, Django, scripts | 2 | +| Go | `cloudflare-go/v4` | CLI tools, microservices | 10 | + +All SDKs are Stainless-generated from OpenAPI spec (consistent APIs). + +## Authentication Methods + +| Method | Security | Use Case | Scope | +|--------|----------|----------|-------| +| **API Token** ✓ | Scoped, rotatable | Production | Per-zone or account | +| API Key + Email | Full account access | Legacy only | Everything | +| User Service Key | Limited | Origin CA certs only | Origin CA | + +**Always use API tokens** for new projects. + +## Rate Limits + +| Limit | Value | +|-------|-------| +| Per user/token | 1200 requests / 5 minutes | +| Per IP | 200 requests / second | +| GraphQL | 320 / 5 minutes (cost-based) | + +## Reading Order + +| Task | Files to Read | +|------|---------------| +| Initialize SDK client | api.md | +| Configure auth/timeout/retry | configuration.md | +| Find usage patterns | patterns.md | +| Debug errors/rate limits | gotchas.md | +| Product-specific APIs | ../workers/, ../r2/, ../kv/, etc. | + +## In This Reference + +- **[api.md](api.md)** - SDK client initialization, pagination, error handling, examples +- **[configuration.md](configuration.md)** - Environment variables, SDK config, Wrangler setup +- **[patterns.md](patterns.md)** - Real-world patterns, batch operations, workflows +- **[gotchas.md](gotchas.md)** - Rate limits, SDK-specific issues, troubleshooting + +## See Also + +- [Cloudflare API Docs](https://developers.cloudflare.com/api/) +- [Bindings Reference](../bindings/) - Workers runtime bindings (preferred over REST API) +- [Wrangler Reference](../wrangler/) - CLI tool for Cloudflare development diff --git a/plugins/cloudflare/skills/cloudflare/references/api/api.md b/plugins/cloudflare/skills/cloudflare/references/api/api.md new file mode 100644 index 0000000..3371014 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api/api.md @@ -0,0 +1,204 @@ +# API Reference + +## Client Initialization + +### TypeScript + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ + apiToken: process.env.CLOUDFLARE_API_TOKEN, +}); +``` + +### Python + +```python +from cloudflare import Cloudflare + +client = Cloudflare(api_token=os.environ.get("CLOUDFLARE_API_TOKEN")) + +# For async: +from cloudflare import AsyncCloudflare +client = AsyncCloudflare(api_token=os.environ["CLOUDFLARE_API_TOKEN"]) +``` + +### Go + +```go +import ( + "github.com/cloudflare/cloudflare-go/v4" + "github.com/cloudflare/cloudflare-go/v4/option" +) + +client := cloudflare.NewClient( + option.WithAPIToken(os.Getenv("CLOUDFLARE_API_TOKEN")), +) +``` + +## Authentication + +### API Token (Recommended) + +**Create token**: Dashboard → My Profile → API Tokens → Create Token + +```bash +export CLOUDFLARE_API_TOKEN='your-token-here' + +curl "https://api.cloudflare.com/client/v4/zones" \ + --header "Authorization: Bearer $CLOUDFLARE_API_TOKEN" +``` + +**Token scopes**: Always use minimal permissions (zone-specific, time-limited). + +### API Key (Legacy) + +```bash +curl "https://api.cloudflare.com/client/v4/zones" \ + --header "X-Auth-Email: user@example.com" \ + --header "X-Auth-Key: $CLOUDFLARE_API_KEY" +``` + +**Not recommended:** Full account access, cannot scope permissions. + +## Auto-Pagination + +All SDKs support automatic pagination for list operations. + +```typescript +// TypeScript: for await...of +for await (const zone of client.zones.list()) { + console.log(zone.id); +} +``` + +```python +# Python: iterator protocol +for zone in client.zones.list(): + print(zone.id) +``` + +```go +// Go: ListAutoPaging +iter := client.Zones.ListAutoPaging(ctx, cloudflare.ZoneListParams{}) +for iter.Next() { + zone := iter.Current() + fmt.Println(zone.ID) +} +``` + +## Error Handling + +```typescript +try { + const zone = await client.zones.get({ zone_id: 'xxx' }); +} catch (err) { + if (err instanceof Cloudflare.NotFoundError) { + // 404 + } else if (err instanceof Cloudflare.RateLimitError) { + // 429 - SDK auto-retries with backoff + } else if (err instanceof Cloudflare.APIError) { + console.log(err.status, err.message); + } +} +``` + +**Common Error Types:** +- `AuthenticationError` (401) - Invalid token +- `PermissionDeniedError` (403) - Insufficient scope +- `NotFoundError` (404) - Resource not found +- `RateLimitError` (429) - Rate limit exceeded +- `InternalServerError` (≥500) - Cloudflare error + +## Zone Management + +```typescript +// List zones +const zones = await client.zones.list({ + account: { id: 'account-id' }, + status: 'active', +}); + +// Create zone +const zone = await client.zones.create({ + account: { id: 'account-id' }, + name: 'example.com', + type: 'full', // or 'partial' +}); + +// Update zone +await client.zones.edit('zone-id', { + paused: false, +}); + +// Delete zone +await client.zones.delete('zone-id'); +``` + +```go +// Go: requires cloudflare.F() wrapper +zone, err := client.Zones.New(ctx, cloudflare.ZoneNewParams{ + Account: cloudflare.F(cloudflare.ZoneNewParamsAccount{ + ID: cloudflare.F("account-id"), + }), + Name: cloudflare.F("example.com"), + Type: cloudflare.F(cloudflare.ZoneNewParamsTypeFull), +}) +``` + +## DNS Management + +```typescript +// Create DNS record +await client.dns.records.create({ + zone_id: 'zone-id', + type: 'A', + name: 'subdomain.example.com', + content: '192.0.2.1', + ttl: 1, // auto + proxied: true, // Orange cloud +}); + +// List DNS records (with auto-pagination) +for await (const record of client.dns.records.list({ + zone_id: 'zone-id', + type: 'A', +})) { + console.log(record.name, record.content); +} + +// Update DNS record +await client.dns.records.update({ + zone_id: 'zone-id', + dns_record_id: 'record-id', + type: 'A', + name: 'subdomain.example.com', + content: '203.0.113.1', + proxied: true, +}); + +// Delete DNS record +await client.dns.records.delete({ + zone_id: 'zone-id', + dns_record_id: 'record-id', +}); +``` + +```python +# Python example +client.dns.records.create( + zone_id="zone-id", + type="A", + name="subdomain.example.com", + content="192.0.2.1", + ttl=1, + proxied=True, +) +``` + +## See Also + +- [configuration.md](./configuration.md) - SDK configuration, environment variables +- [patterns.md](./patterns.md) - Real-world patterns and workflows +- [gotchas.md](./gotchas.md) - Rate limits, troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/api/configuration.md b/plugins/cloudflare/skills/cloudflare/references/api/configuration.md new file mode 100644 index 0000000..4d4a299 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api/configuration.md @@ -0,0 +1,160 @@ +# Configuration + +## Environment Variables + +### Set Variables + +| Platform | Command | +|----------|---------| +| Linux/macOS | `export CLOUDFLARE_API_TOKEN='token'` | +| PowerShell | `$env:CLOUDFLARE_API_TOKEN = 'token'` | +| Windows CMD | `set CLOUDFLARE_API_TOKEN=token` | + +**Security:** Never commit tokens. Use `.env` files (gitignored) or secret managers. + +### .env File Pattern + +```bash +# .env (add to .gitignore) +CLOUDFLARE_API_TOKEN=your-token-here +CLOUDFLARE_ACCOUNT_ID=your-account-id +``` + +```typescript +// TypeScript +import 'dotenv/config'; + +const client = new Cloudflare({ + apiToken: process.env.CLOUDFLARE_API_TOKEN, +}); +``` + +```python +# Python +from dotenv import load_dotenv +load_dotenv() + +client = Cloudflare(api_token=os.environ["CLOUDFLARE_API_TOKEN"]) +``` + +## SDK Configuration + +### TypeScript + +```typescript +const client = new Cloudflare({ + apiToken: process.env.CLOUDFLARE_API_TOKEN, + timeout: 120000, // 2 min (default 60s), in milliseconds + maxRetries: 5, // default 2 + baseURL: 'https://...', // proxy (rare) +}); + +// Per-request overrides +await client.zones.get( + { zone_id: 'zone-id' }, + { timeout: 5000, maxRetries: 0 } +); +``` + +### Python + +```python +client = Cloudflare( + api_token=os.environ["CLOUDFLARE_API_TOKEN"], + timeout=120, # seconds (default 60) + max_retries=5, # default 2 + base_url="https://...", # proxy (rare) +) + +# Per-request overrides +client.with_options(timeout=5, max_retries=0).zones.get(zone_id="zone-id") +``` + +### Go + +```go +client := cloudflare.NewClient( + option.WithAPIToken(os.Getenv("CLOUDFLARE_API_TOKEN")), + option.WithMaxRetries(5), // default 10 (higher than TS/Python) + option.WithRequestTimeout(2 * time.Minute), // default 60s + option.WithBaseURL("https://..."), // proxy (rare) +) + +// Per-request overrides +client.Zones.Get(ctx, "zone-id", option.WithMaxRetries(0)) +``` + +## Configuration Options + +| Option | TypeScript | Python | Go | Default | +|--------|-----------|--------|-----|---------| +| Timeout | `timeout` (ms) | `timeout` (s) | `WithRequestTimeout` | 60s | +| Retries | `maxRetries` | `max_retries` | `WithMaxRetries` | 2 (Go: 10) | +| Base URL | `baseURL` | `base_url` | `WithBaseURL` | api.cloudflare.com | + +**Note:** Go SDK has higher default retries (10) than TypeScript/Python (2). + +## Timeout Configuration + +**When to increase:** +- Large zone transfers +- Bulk DNS operations +- Worker script uploads + +```typescript +const client = new Cloudflare({ + timeout: 300000, // 5 minutes +}); +``` + +## Retry Configuration + +**When to increase:** Rate-limit-heavy workflows, flaky network + +**When to decrease:** Fast-fail requirements, user-facing requests + +```typescript +// Increase retries for batch operations +const client = new Cloudflare({ maxRetries: 10 }); + +// Disable retries for fast-fail +const fastClient = new Cloudflare({ maxRetries: 0 }); +``` + +## Wrangler CLI Integration + +```bash +# Configure authentication +wrangler login +# Or +export CLOUDFLARE_API_TOKEN='token' + +# Common commands that use API +wrangler deploy # Uploads worker via API +wrangler kv:key put # KV operations +wrangler r2 bucket create # R2 operations +wrangler d1 execute # D1 operations +wrangler pages deploy # Pages operations + +# Get API configuration +wrangler whoami # Shows authenticated user +``` + +### wrangler.toml + +```toml +name = "my-worker" +main = "src/index.ts" +compatibility_date = "2024-01-01" +account_id = "your-account-id" + +# Can also use env vars: +# CLOUDFLARE_ACCOUNT_ID +# CLOUDFLARE_API_TOKEN +``` + +## See Also + +- [api.md](./api.md) - Client initialization, authentication +- [gotchas.md](./gotchas.md) - Rate limits, timeout errors +- [Wrangler Reference](../wrangler/) - CLI tool details diff --git a/plugins/cloudflare/skills/cloudflare/references/api/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/api/gotchas.md new file mode 100644 index 0000000..e6666dc --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api/gotchas.md @@ -0,0 +1,225 @@ +# Gotchas & Troubleshooting + +## Rate Limits & 429 Errors + +**Actual Limits:** +- **1200 requests / 5 minutes** per user/token (global) +- **200 requests / second** per IP address +- **GraphQL: 320 / 5 minutes** (cost-based) + +**SDK Behavior:** +- Auto-retry with exponential backoff (default 2 retries, Go: 10) +- Respects `Retry-After` header +- Throws `RateLimitError` after exhausting retries + +**Solution:** + +```typescript +// Increase retries for rate-limit-heavy workflows +const client = new Cloudflare({ maxRetries: 5 }); + +// Add application-level throttling +import pLimit from 'p-limit'; +const limit = pLimit(10); // Max 10 concurrent requests +``` + +## SDK-Specific Issues + +### Go: Required Field Wrapper + +**Problem:** Go SDK requires `cloudflare.F()` wrapper for optional fields. + +```go +// ❌ WRONG - Won't compile or send field +client.Zones.New(ctx, cloudflare.ZoneNewParams{ + Name: "example.com", +}) + +// ✅ CORRECT +client.Zones.New(ctx, cloudflare.ZoneNewParams{ + Name: cloudflare.F("example.com"), + Account: cloudflare.F(cloudflare.ZoneNewParamsAccount{ + ID: cloudflare.F("account-id"), + }), +}) +``` + +**Why:** Distinguishes between zero value, null, and omitted fields. + +### Python: Async vs Sync Clients + +**Problem:** Using sync client in async context or vice versa. + +```python +# ❌ WRONG - Can't await sync client +from cloudflare import Cloudflare +client = Cloudflare() +await client.zones.list() # TypeError + +# ✅ CORRECT - Use AsyncCloudflare +from cloudflare import AsyncCloudflare +client = AsyncCloudflare() +await client.zones.list() +``` + +## Token Permission Errors (403) + +**Problem:** API returns 403 Forbidden despite valid token. + +**Cause:** Token lacks required permissions (scope). + +**Scopes Required:** + +| Operation | Required Scope | +|-----------|----------------| +| List zones | Zone:Read (zone-level or account-level) | +| Create zone | Zone:Edit (account-level) | +| Edit DNS | DNS:Edit (zone-level) | +| Deploy Worker | Workers Script:Edit (account-level) | +| Read KV | Workers KV Storage:Read | +| Write KV | Workers KV Storage:Edit | + +**Solution:** Re-create token with correct permissions in Dashboard → My Profile → API Tokens. + +## Pagination Truncation + +**Problem:** Only getting first 20 results (default page size). + +**Solution:** Use auto-pagination iterators. + +```typescript +// ❌ WRONG - Only first page (20 items) +const page = await client.zones.list(); + +// ✅ CORRECT - All results +const zones = []; +for await (const zone of client.zones.list()) { + zones.push(zone); +} +``` + +## Workers Subrequests + +**Problem:** Rate limit hit faster than expected in Workers. + +**Cause:** Workers subrequests count as separate API calls. + +**Solution:** Use bindings instead of REST API in Workers (see ../bindings/). + +```typescript +// ❌ WRONG - REST API in Workers (counts against rate limit) +const client = new Cloudflare({ apiToken: env.CLOUDFLARE_API_TOKEN }); +const zones = await client.zones.list(); + +// ✅ CORRECT - Use bindings (no rate limit) +// Access via env.MY_BINDING +``` + +## Authentication Errors (401) + +**Problem:** "Authentication failed" or "Invalid token" + +**Causes:** +- Token expired +- Token deleted/revoked +- Token not set in environment +- Wrong token format + +**Solution:** + +```typescript +// Verify token is set +if (!process.env.CLOUDFLARE_API_TOKEN) { + throw new Error('CLOUDFLARE_API_TOKEN not set'); +} + +// Test token +const user = await client.user.tokens.verify(); +console.log('Token valid:', user.status); +``` + +## Timeout Errors + +**Problem:** Request times out (default 60s). + +**Cause:** Large operations (bulk DNS, zone transfers). + +**Solution:** Increase timeout or split operations. + +```typescript +// Increase timeout +const client = new Cloudflare({ + timeout: 300000, // 5 minutes +}); + +// Or split operations +const batchSize = 100; +for (let i = 0; i < records.length; i += batchSize) { + const batch = records.slice(i, i + batchSize); + await processBatch(batch); +} +``` + +## Zone Not Found (404) + +**Problem:** Zone ID valid but returns 404. + +**Causes:** +- Zone not in account associated with token +- Zone deleted +- Wrong zone ID format + +**Solution:** + +```typescript +// List all zones to find correct ID +for await (const zone of client.zones.list()) { + console.log(zone.id, zone.name); +} +``` + +## Limits Reference + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| API rate limit | 1200/5min | Per user/token | +| IP rate limit | 200/sec | Per IP | +| GraphQL rate limit | 320/5min | Cost-based | +| Parallel requests (recommended) | < 10 | Avoid overwhelming API | +| Default page size | 20 | Use auto-pagination | +| Max page size | 50 | Some endpoints | + +## Best Practices + +**Security:** +- Never commit tokens +- Use minimal permissions +- Rotate tokens regularly +- Set token expiration + +**Performance:** +- Batch operations +- Use pagination wisely +- Cache responses +- Handle rate limits + +**Code Organization:** + +```typescript +// Create reusable client instance +export const cfClient = new Cloudflare({ + apiToken: process.env.CLOUDFLARE_API_TOKEN, + maxRetries: 5, +}); + +// Wrap common operations +export async function getZoneDetails(zoneId: string) { + return await cfClient.zones.get({ zone_id: zoneId }); +} +``` + +## See Also + +- [api.md](./api.md) - Error types, authentication +- [configuration.md](./configuration.md) - Timeout/retry configuration +- [patterns.md](./patterns.md) - Error handling patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/api/patterns.md b/plugins/cloudflare/skills/cloudflare/references/api/patterns.md new file mode 100644 index 0000000..3c7c693 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/api/patterns.md @@ -0,0 +1,204 @@ +# Common Patterns + +## List All with Auto-Pagination + +**Problem:** API returns paginated results. Default page size is 20. + +**Solution:** Use SDK auto-pagination to iterate all results. + +```typescript +// TypeScript +for await (const zone of client.zones.list()) { + console.log(zone.name); +} +``` + +```python +# Python +for zone in client.zones.list(): + print(zone.name) +``` + +```go +// Go +iter := client.Zones.ListAutoPaging(ctx, cloudflare.ZoneListParams{}) +for iter.Next() { + fmt.Println(iter.Current().Name) +} +``` + +## Error Handling with Retry + +**Problem:** Rate limits (429) and transient errors need retry. + +**Solution:** SDKs auto-retry with exponential backoff. Customize as needed. + +```typescript +// Increase retries for rate-limit-heavy operations +const client = new Cloudflare({ maxRetries: 5 }); + +try { + const zone = await client.zones.create({ /* ... */ }); +} catch (err) { + if (err instanceof Cloudflare.RateLimitError) { + // Already retried 5 times with backoff + const retryAfter = err.headers['retry-after']; + console.log(`Rate limited. Retry after ${retryAfter}s`); + } +} +``` + +## Batch Parallel Operations + +**Problem:** Need to create multiple resources quickly. + +**Solution:** Use `Promise.all()` for parallel requests (respect rate limits). + +```typescript +// Create multiple DNS records in parallel +const records = ['www', 'api', 'cdn'].map(subdomain => + client.dns.records.create({ + zone_id: 'zone-id', + type: 'A', + name: `${subdomain}.example.com`, + content: '192.0.2.1', + }) +); +await Promise.all(records); +``` + +**Controlled concurrency** (avoid rate limits): + +```typescript +import pLimit from 'p-limit'; +const limit = pLimit(10); // Max 10 concurrent + +const subdomains = ['www', 'api', 'cdn', /* many more */]; +const records = subdomains.map(subdomain => + limit(() => client.dns.records.create({ + zone_id: 'zone-id', + type: 'A', + name: `${subdomain}.example.com`, + content: '192.0.2.1', + })) +); +await Promise.all(records); +``` + +## Zone CRUD Workflow + +```typescript +// Create +const zone = await client.zones.create({ + account: { id: 'account-id' }, + name: 'example.com', + type: 'full', +}); + +// Read +const fetched = await client.zones.get({ zone_id: zone.id }); + +// Update +await client.zones.edit(zone.id, { paused: false }); + +// Delete +await client.zones.delete(zone.id); +``` + +## DNS Bulk Update + +```typescript +// Fetch all A records +const records = []; +for await (const record of client.dns.records.list({ + zone_id: 'zone-id', + type: 'A', +})) { + records.push(record); +} + +// Update all to new IP +await Promise.all(records.map(record => + client.dns.records.update({ + zone_id: 'zone-id', + dns_record_id: record.id, + type: 'A', + name: record.name, + content: '203.0.113.1', // New IP + proxied: record.proxied, + ttl: record.ttl, + }) +)); +``` + +## Filter and Collect Results + +```typescript +// Find all proxied A records +const proxiedRecords = []; +for await (const record of client.dns.records.list({ + zone_id: 'zone-id', + type: 'A', +})) { + if (record.proxied) { + proxiedRecords.push(record); + } +} +``` + +## Error Recovery Pattern + +```typescript +async function createZoneWithRetry(name: string, maxAttempts = 3) { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await client.zones.create({ + account: { id: 'account-id' }, + name, + type: 'full', + }); + } catch (err) { + if (err instanceof Cloudflare.RateLimitError && attempt < maxAttempts) { + const retryAfter = parseInt(err.headers['retry-after'] || '5'); + console.log(`Rate limited, waiting ${retryAfter}s (retry ${attempt}/${maxAttempts})`); + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)); + } else { + throw err; + } + } + } +} +``` + +## Conditional Update Pattern + +```typescript +// Only update if zone is active +const zone = await client.zones.get({ zone_id: 'zone-id' }); +if (zone.status === 'active') { + await client.zones.edit(zone.id, { paused: false }); +} +``` + +## Batch with Error Handling + +```typescript +// Process multiple zones, continue on errors +const results = await Promise.allSettled( + zoneIds.map(id => client.zones.get({ zone_id: id })) +); + +results.forEach((result, i) => { + if (result.status === 'fulfilled') { + console.log(`Zone ${i}: ${result.value.name}`); + } else { + console.error(`Zone ${i} failed:`, result.reason.message); + } +}); +``` + +## See Also + +- [api.md](./api.md) - SDK client initialization, basic operations +- [gotchas.md](./gotchas.md) - Rate limits, common errors +- [configuration.md](./configuration.md) - SDK configuration options diff --git a/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/README.md b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/README.md new file mode 100644 index 0000000..ef5539d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/README.md @@ -0,0 +1,90 @@ +# Cloudflare Argo Smart Routing Skill Reference + +## Overview + +Cloudflare Argo Smart Routing is a performance optimization service that detects real-time network issues and routes web traffic across the most efficient network path. It continuously monitors network conditions and intelligently routes traffic through the fastest, most reliable routes in Cloudflare's network. + +**Note on Smart Shield:** Argo Smart Routing is being integrated into Cloudflare's Smart Shield product for enhanced DDoS protection and performance. Existing Argo customers maintain full functionality with gradual migration to Smart Shield features. + +## Quick Start + +### Enable via cURL +```bash +curl -X PATCH "https://api.cloudflare.com/client/v4/zones/{zone_id}/argo/smart_routing" \ + -H "Authorization: Bearer YOUR_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"value": "on"}' +``` + +### Enable via TypeScript SDK +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: process.env.CLOUDFLARE_API_TOKEN }); + +const result = await client.argo.smartRouting.edit({ + zone_id: 'your-zone-id', + value: 'on', +}); + +console.log(`Argo enabled: ${result.value}`); +``` + +## Core Concepts + +### What It Does +- **Intelligent routing**: Detects congestion, outages, packet loss in real-time +- **Global optimization**: Routes across 300+ Cloudflare data centers +- **Automatic failover**: Switches paths when issues detected (typically <1s) +- **Works with existing setup**: No origin changes required + +### Billing Model +- Usage-based: Charged per GB of traffic (excluding DDoS/WAF mitigated traffic) +- Requires billing configuration before enabling +- Available on Enterprise+ plans (check zone eligibility) + +### When to Use +- **High-traffic production sites** with global user base +- **Latency-sensitive applications** (APIs, real-time services) +- **Sites behind Cloudflare proxy** (orange-clouded DNS records) +- **Combined with Tiered Cache** for maximum performance gains + +### When NOT to Use +- Development/staging environments (cost control) +- Low-traffic sites (<1TB/month) where cost may exceed benefit +- Sites with primarily single-region traffic + +## Should I Enable Argo? + +| Your Situation | Recommendation | +|----------------|----------------| +| Global production app, >1TB/month traffic | ✅ Enable - likely ROI positive | +| Enterprise plan, latency-critical APIs | ✅ Enable - performance matters | +| Regional site, <100GB/month traffic | ⚠️ Evaluate - cost may not justify | +| Development/staging environment | ❌ Disable - use in production only | +| Not yet configured billing | ❌ Configure billing first | + +## Reading Order by Task + +| Your Goal | Start With | Then Read | +|-----------|------------|-----------| +| Enable Argo for first time | Quick Start above → [configuration.md](configuration.md) | [gotchas.md](gotchas.md) | +| Use TypeScript/Python SDK | [api.md](api.md) | [patterns.md](patterns.md) | +| Terraform/IaC setup | [configuration.md](configuration.md) | - | +| Enable for Spectrum TCP app | [patterns.md](patterns.md) → Spectrum section | [api.md](api.md) | +| Troubleshoot enablement issue | [gotchas.md](gotchas.md) | [api.md](api.md) | +| Manage billing/usage | [patterns.md](patterns.md) → Billing section | [gotchas.md](gotchas.md) | + +## In This Reference + +- **[api.md](api.md)** - API endpoints, SDK methods, error handling, Python/TypeScript examples +- **[configuration.md](configuration.md)** - Terraform setup, environment config, billing configuration +- **[patterns.md](patterns.md)** - Tiered Cache integration, Spectrum TCP apps, billing management, validation patterns +- **[gotchas.md](gotchas.md)** - Common errors, permission issues, limits, best practices + +## See Also + +- [Cloudflare Argo Smart Routing Docs](https://developers.cloudflare.com/argo-smart-routing/) +- [Cloudflare Smart Shield](https://developers.cloudflare.com/smart-shield/) +- [Spectrum Documentation](https://developers.cloudflare.com/spectrum/) +- [Tiered Cache](https://developers.cloudflare.com/cache/how-to/tiered-cache/) diff --git a/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/api.md b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/api.md new file mode 100644 index 0000000..5cd8f5e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/api.md @@ -0,0 +1,240 @@ +## API Reference + +**Note on Smart Shield:** Argo Smart Routing is being integrated into Cloudflare's Smart Shield product. API endpoints remain stable; existing integrations continue to work without changes. + +### Base Endpoint +``` +https://api.cloudflare.com/client/v4 +``` + +### Authentication +Use API tokens with Zone:Argo Smart Routing:Edit permissions: + +```bash +# Headers required +X-Auth-Email: user@example.com +Authorization: Bearer YOUR_API_TOKEN +``` + +### Get Argo Smart Routing Status + +**Endpoint:** `GET /zones/{zone_id}/argo/smart_routing` + +**Description:** Retrieves current Argo Smart Routing enablement status. + +**cURL Example:** +```bash +curl -X GET "https://api.cloudflare.com/client/v4/zones/{zone_id}/argo/smart_routing" \ + -H "Authorization: Bearer YOUR_API_TOKEN" \ + -H "Content-Type: application/json" +``` + +**Response:** +```json +{ + "result": { + "id": "smart_routing", + "value": "on", + "editable": true, + "modified_on": "2024-01-11T12:00:00Z" + }, + "success": true, + "errors": [], + "messages": [] +} +``` + +**TypeScript SDK Example:** +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ + apiToken: process.env.CLOUDFLARE_API_TOKEN +}); + +const status = await client.argo.smartRouting.get({ zone_id: 'your-zone-id' }); +console.log(`Argo status: ${status.value}, editable: ${status.editable}`); +``` + +**Python SDK Example:** +```python +from cloudflare import Cloudflare + +client = Cloudflare(api_token=os.environ.get('CLOUDFLARE_API_TOKEN')) + +status = client.argo.smart_routing.get(zone_id='your-zone-id') +print(f"Argo status: {status.value}, editable: {status.editable}") +``` + +### Update Argo Smart Routing Status + +**Endpoint:** `PATCH /zones/{zone_id}/argo/smart_routing` + +**Description:** Enable or disable Argo Smart Routing for a zone. + +**Request Body:** +```json +{ + "value": "on" // or "off" +} +``` + +**cURL Example:** +```bash +curl -X PATCH "https://api.cloudflare.com/client/v4/zones/{zone_id}/argo/smart_routing" \ + -H "Authorization: Bearer YOUR_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"value": "on"}' +``` + +**TypeScript SDK Example:** +```typescript +const result = await client.argo.smartRouting.edit({ + zone_id: 'your-zone-id', + value: 'on', +}); +console.log(`Updated: ${result.value} at ${result.modified_on}`); +``` + +**Python SDK Example:** +```python +result = client.argo.smart_routing.edit( + zone_id='your-zone-id', + value='on' +) +print(f"Updated: {result.value} at {result.modified_on}") +``` + +## Checking Editability Before Updates + +**Critical:** Always check the `editable` field before attempting to enable/disable Argo. When `editable: false`, the zone has restrictions (billing not configured, insufficient permissions, or plan limitations). + +**Pattern:** +```typescript +async function safelyEnableArgo(client: Cloudflare, zoneId: string): Promise { + const status = await client.argo.smartRouting.get({ zone_id: zoneId }); + + if (!status.editable) { + console.error('Cannot modify Argo: editable=false (check billing/permissions)'); + return false; + } + + if (status.value === 'on') { + console.log('Argo already enabled'); + return true; + } + + await client.argo.smartRouting.edit({ zone_id: zoneId, value: 'on' }); + console.log('Argo enabled successfully'); + return true; +} +``` + +**Python Pattern:** +```python +def safely_enable_argo(client: Cloudflare, zone_id: str) -> bool: + status = client.argo.smart_routing.get(zone_id=zone_id) + + if not status.editable: + print('Cannot modify Argo: editable=false (check billing/permissions)') + return False + + if status.value == 'on': + print('Argo already enabled') + return True + + client.argo.smart_routing.edit(zone_id=zone_id, value='on') + print('Argo enabled successfully') + return True +``` + +## Error Handling + +The TypeScript SDK provides typed error classes for robust error handling: + +```typescript +import Cloudflare from 'cloudflare'; +import { APIError, APIConnectionError, RateLimitError } from 'cloudflare'; + +async function enableArgoWithErrorHandling(client: Cloudflare, zoneId: string) { + try { + const result = await client.argo.smartRouting.edit({ + zone_id: zoneId, + value: 'on', + }); + return result; + } catch (error) { + if (error instanceof RateLimitError) { + console.error('Rate limited. Retry after:', error.response?.headers.get('retry-after')); + // Implement exponential backoff + } else if (error instanceof APIError) { + console.error('API error:', error.status, error.message); + if (error.status === 403) { + console.error('Permission denied - check API token scopes'); + } else if (error.status === 400) { + console.error('Bad request - verify zone_id and payload'); + } + } else if (error instanceof APIConnectionError) { + console.error('Connection failed:', error.message); + // Retry with exponential backoff + } else { + console.error('Unexpected error:', error); + } + throw error; + } +} +``` + +**Python Error Handling:** +```python +from cloudflare import Cloudflare, APIError, RateLimitError + +def enable_argo_with_error_handling(client: Cloudflare, zone_id: str): + try: + result = client.argo.smart_routing.edit(zone_id=zone_id, value='on') + return result + except RateLimitError as e: + print(f"Rate limited. Retry after: {e.response.headers.get('retry-after')}") + raise + except APIError as e: + print(f"API error: {e.status} - {e.message}") + if e.status == 403: + print('Permission denied - check API token scopes') + elif e.status == 400: + print('Bad request - verify zone_id and payload') + raise + except Exception as e: + print(f"Unexpected error: {e}") + raise +``` + +## Response Schema + +All Argo Smart Routing API responses follow this structure: + +```typescript +interface ArgoSmartRoutingResponse { + result: { + id: 'smart_routing'; + value: 'on' | 'off'; + editable: boolean; + modified_on: string; // ISO 8601 timestamp + }; + success: boolean; + errors: Array<{ + code: number; + message: string; + }>; + messages: Array; +} +``` + +## Key Response Fields + +| Field | Type | Description | +|-------|------|-------------| +| `value` | `"on" \| "off"` | Current enablement status | +| `editable` | `boolean` | Whether changes are allowed (check before PATCH) | +| `modified_on` | `string` | ISO timestamp of last modification | +| `success` | `boolean` | Whether request succeeded | +| `errors` | `Array` | Error details if `success: false` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/configuration.md b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/configuration.md new file mode 100644 index 0000000..ba94d38 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/configuration.md @@ -0,0 +1,197 @@ +## Configuration Management + +**Note on Smart Shield Evolution:** Argo Smart Routing is being integrated into Smart Shield. Configuration methods below remain valid; Terraform and IaC patterns unchanged. + +### Infrastructure as Code (Terraform) + +```hcl +# terraform/argo.tf +# Note: Use Cloudflare Terraform provider + +resource "cloudflare_argo" "example" { + zone_id = var.zone_id + smart_routing = "on" + tiered_caching = "on" +} + +variable "zone_id" { + description = "Cloudflare Zone ID" + type = string +} + +output "argo_enabled" { + value = cloudflare_argo.example.smart_routing + description = "Argo Smart Routing status" +} +``` + +### Environment-Based Configuration + +```typescript +// config/argo.ts +interface ArgoEnvironmentConfig { + enabled: boolean; + tieredCache: boolean; + monitoring: { + usageAlerts: boolean; + threshold: number; + }; +} + +const configs: Record = { + production: { + enabled: true, + tieredCache: true, + monitoring: { + usageAlerts: true, + threshold: 1000, // GB + }, + }, + staging: { + enabled: true, + tieredCache: false, + monitoring: { + usageAlerts: false, + threshold: 100, // GB + }, + }, + development: { + enabled: false, + tieredCache: false, + monitoring: { + usageAlerts: false, + threshold: 0, + }, + }, +}; + +export function getArgoConfig(env: string): ArgoEnvironmentConfig { + return configs[env] || configs.development; +} +``` + +### Pulumi Configuration + +```typescript +// pulumi/argo.ts +import * as cloudflare from '@pulumi/cloudflare'; + +const zone = new cloudflare.Zone('example-zone', { + zone: 'example.com', + plan: 'enterprise', +}); + +const argoSettings = new cloudflare.Argo('argo-config', { + zoneId: zone.id, + smartRouting: 'on', + tieredCaching: 'on', +}); + +export const argoEnabled = argoSettings.smartRouting; +export const zoneId = zone.id; +``` + +## Billing Configuration + +Before enabling Argo Smart Routing, ensure billing is configured for the account: + +**Prerequisites:** +1. Valid payment method on file +2. Enterprise or higher plan +3. Zone must have billing enabled + +**Check Billing Status via Dashboard:** +1. Navigate to Account → Billing +2. Verify payment method configured +3. Check zone subscription status + +**Note:** Attempting to enable Argo without billing configured will result in `editable: false` in API responses. + +## Environment Variable Setup + +**Required Environment Variables:** +```bash +# .env +CLOUDFLARE_API_TOKEN=your_api_token_here +CLOUDFLARE_ZONE_ID=your_zone_id_here +CLOUDFLARE_ACCOUNT_ID=your_account_id_here + +# Optional +ARGO_ENABLED=true +ARGO_TIERED_CACHE=true +``` + +**TypeScript Configuration Loader:** +```typescript +// config/env.ts +import { z } from 'zod'; + +const envSchema = z.object({ + CLOUDFLARE_API_TOKEN: z.string().min(1), + CLOUDFLARE_ZONE_ID: z.string().min(1), + CLOUDFLARE_ACCOUNT_ID: z.string().min(1), + ARGO_ENABLED: z.string().optional().default('false'), + ARGO_TIERED_CACHE: z.string().optional().default('false'), +}); + +export const env = envSchema.parse(process.env); + +export const argoConfig = { + enabled: env.ARGO_ENABLED === 'true', + tieredCache: env.ARGO_TIERED_CACHE === 'true', +}; +``` + +## CI/CD Integration + +**GitHub Actions Example:** +```yaml +# .github/workflows/deploy-argo.yml +name: Deploy Argo Configuration + +on: + push: + branches: [main] + paths: + - 'terraform/argo.tf' + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + + - name: Terraform Init + run: terraform init + working-directory: ./terraform + + - name: Terraform Apply + run: terraform apply -auto-approve + working-directory: ./terraform + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + TF_VAR_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }} +``` + +## Enterprise Preview Program + +For early access to Argo Smart Routing features and Smart Shield integration: + +**Eligibility:** +- Enterprise plan customers +- Active Cloudflare support contract +- Production traffic >100GB/month + +**How to Join:** +1. Contact Cloudflare account team or support +2. Request Argo/Smart Shield preview access +3. Receive preview zone configuration + +**Preview Features:** +- Enhanced analytics and reporting +- Smart Shield DDoS integration +- Advanced routing policies +- Priority support for routing issues \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/gotchas.md new file mode 100644 index 0000000..8d011d3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/gotchas.md @@ -0,0 +1,111 @@ +## Best Practices Summary + +**Smart Shield Note:** Argo Smart Routing evolving into Smart Shield. Best practices below remain applicable; monitor Cloudflare changelog for Smart Shield updates. + +1. **Always check editability** before attempting to enable/disable Argo +2. **Set up billing notifications** to avoid unexpected costs +3. **Combine with Tiered Cache** for maximum performance benefit +4. **Use in production only** - disable for dev/staging to control costs +5. **Monitor analytics** - require 500+ requests in 48h for detailed metrics +6. **Handle errors gracefully** - check for billing, permissions, zone compatibility +7. **Test configuration changes** in staging before production +8. **Use TypeScript SDK** for type safety and better developer experience +9. **Implement retry logic** for API calls in production systems +10. **Document zone-specific settings** for team visibility + +## Common Errors + +### "Argo unavailable" + +**Problem:** API returns error "Argo Smart Routing is unavailable for this zone" + +**Cause:** Zone not eligible or billing not set up + +**Solution:** +1. Verify zone has Enterprise or higher plan +2. Check billing is configured in Account → Billing +3. Ensure payment method is valid and current +4. Contact Cloudflare support if eligibility unclear + +### "Cannot enable/disable" + +**Problem:** API call succeeds but status remains unchanged, or `editable: false` in GET response + +**Cause:** Insufficient permissions or zone restrictions + +**Solution:** +1. Check API token has `Zone:Argo Smart Routing:Edit` permission +2. Verify `editable: true` in GET response before attempting PATCH +3. If `editable: false`, check: + - Billing configured for account + - Zone plan includes Argo (Enterprise+) + - No active zone holds or suspensions + - API token has correct scopes + +### `editable: false` Error + +**Problem:** GET request returns `"editable": false`, preventing enable/disable + +**Cause:** Zone-level restrictions from billing, plan, or permissions + +**Solution Pattern:** +```typescript +const status = await client.argo.smartRouting.get({ zone_id: zoneId }); + +if (!status.editable) { + // Don't attempt to modify - will fail + console.error('Cannot modify Argo settings:'); + console.error('- Check billing is configured'); + console.error('- Verify zone has Enterprise+ plan'); + console.error('- Confirm API token has Edit permission'); + throw new Error('Argo is not editable for this zone'); +} + +// Safe to proceed with enable/disable +await client.argo.smartRouting.edit({ zone_id: zoneId, value: 'on' }); +``` + +### Rate Limiting + +**Problem:** `429 Too Many Requests` error from API + +**Cause:** Exceeded API rate limits (typically 1200 requests per 5 minutes) + +**Solution:** +```typescript +import { RateLimitError } from 'cloudflare'; + +try { + await client.argo.smartRouting.edit({ zone_id: zoneId, value: 'on' }); +} catch (error) { + if (error instanceof RateLimitError) { + const retryAfter = error.response?.headers.get('retry-after'); + console.log(`Rate limited. Retry after ${retryAfter} seconds`); + + // Implement exponential backoff + await new Promise(resolve => setTimeout(resolve, (retryAfter || 60) * 1000)); + // Retry request + } +} +``` + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Min requests for analytics | 500 in 48h | For detailed metrics via GraphQL | +| Zones supported | Enterprise+ | Check zone plan in dashboard | +| Billing requirement | Must be configured | Before enabling; verify payment method | +| API rate limit | 1200 req / 5 min | Per API token across all endpoints | +| Spectrum apps | No hard limit | Each app can enable Argo independently | +| Traffic counting | Proxied only | Only orange-clouded DNS records count | +| DDoS/WAF exemption | Yes | Mitigated traffic excluded from billing | +| Analytics latency | 1-5 minutes | Real-time metrics not available | + +## Additional Resources + +- [Official Argo Smart Routing Docs](https://developers.cloudflare.com/argo-smart-routing/) +- [Cloudflare Smart Shield](https://developers.cloudflare.com/smart-shield/) +- [API Authentication](https://developers.cloudflare.com/fundamentals/api/get-started/create-token/) +- [Cloudflare TypeScript SDK](https://github.com/cloudflare/cloudflare-typescript) +- [Cloudflare Python SDK](https://github.com/cloudflare/cloudflare-python) diff --git a/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/patterns.md b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/patterns.md new file mode 100644 index 0000000..1173b2d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/argo-smart-routing/patterns.md @@ -0,0 +1,104 @@ +# Integration Patterns + +## Enable Argo + Tiered Cache + +```typescript +async function enableOptimalPerformance(client: Cloudflare, zoneId: string) { + await Promise.all([ + client.argo.smartRouting.edit({ zone_id: zoneId, value: 'on' }), + client.argo.tieredCaching.edit({ zone_id: zoneId, value: 'on' }), + ]); +} +``` + +**Flow:** Visitor → Edge (Lower-Tier) → [Cache Miss] → Upper-Tier → [Cache Miss + Argo] → Origin + +**Impact:** Argo ~30% latency reduction + Tiered Cache 50-80% origin offload + +## Usage Analytics (GraphQL) + +```graphql +query ArgoAnalytics($zoneTag: string!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + httpRequestsAdaptiveGroups(limit: 1000) { + sum { argoBytes, bytes } + } + } + } +} +``` + +**Billing:** ~$0.10/GB. DDoS-mitigated and WAF-blocked traffic NOT charged. + +## Spectrum TCP Integration + +Enable Argo for non-HTTP traffic (databases, game servers, IoT): + +```typescript +// Update existing app +await client.spectrum.apps.update(appId, { zone_id: zoneId, argo_smart_routing: true }); + +// Create new app with Argo +await client.spectrum.apps.create({ + zone_id: zoneId, + dns: { type: 'CNAME', name: 'tcp.example.com' }, + origin_direct: ['tcp://origin.example.com:3306'], + protocol: 'tcp/3306', + argo_smart_routing: true, +}); +``` + +**Use cases:** MySQL/PostgreSQL (3306/5432), game servers, MQTT (1883), SSH (22) + +## Pre-Flight Validation + +```typescript +async function validateArgoEligibility(client: Cloudflare, zoneId: string) { + const status = await client.argo.smartRouting.get({ zone_id: zoneId }); + const zone = await client.zones.get({ zone_id: zoneId }); + + const issues: string[] = []; + if (!status.editable) issues.push('Zone not editable'); + if (['free', 'pro'].includes(zone.plan.legacy_id)) issues.push('Requires Business+ plan'); + if (zone.status !== 'active') issues.push('Zone not active'); + + return { canEnable: issues.length === 0, issues }; +} +``` + +## Post-Enable Verification + +```typescript +async function verifyArgoEnabled(client: Cloudflare, zoneId: string): Promise { + await new Promise(r => setTimeout(r, 2000)); // Wait for propagation + const status = await client.argo.smartRouting.get({ zone_id: zoneId }); + return status.value === 'on'; +} +``` + +## Full Setup Pattern + +```typescript +async function setupArgo(client: Cloudflare, zoneId: string) { + // 1. Validate + const { canEnable, issues } = await validateArgoEligibility(client, zoneId); + if (!canEnable) throw new Error(issues.join(', ')); + + // 2. Enable both features + await Promise.all([ + client.argo.smartRouting.edit({ zone_id: zoneId, value: 'on' }), + client.argo.tieredCaching.edit({ zone_id: zoneId, value: 'on' }), + ]); + + // 3. Verify + const [argo, cache] = await Promise.all([ + client.argo.smartRouting.get({ zone_id: zoneId }), + client.argo.tieredCaching.get({ zone_id: zoneId }), + ]); + + return { argo: argo.value === 'on', tieredCache: cache.value === 'on' }; +} +``` + +**When to combine:** High-traffic sites (>1TB/mo), global users, cacheable content. diff --git a/plugins/cloudflare/skills/cloudflare/references/bindings/README.md b/plugins/cloudflare/skills/cloudflare/references/bindings/README.md new file mode 100644 index 0000000..fc3e8eb --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bindings/README.md @@ -0,0 +1,122 @@ +# Cloudflare Bindings Skill Reference + +Expert guidance on Cloudflare Workers Bindings - the runtime APIs that connect Workers to Cloudflare platform resources. + +## What Are Bindings? + +Bindings are how Workers access Cloudflare resources (storage, compute, services) via the `env` object. They're configured in `wrangler.jsonc`, type-safe via TypeScript, and zero-overhead at runtime. + +## Reading Order + +1. **This file** - Binding catalog and selection guide +2. **[api.md](api.md)** - TypeScript types and env access patterns +3. **[configuration.md](configuration.md)** - Complete wrangler.jsonc examples +4. **[patterns.md](patterns.md)** - Best practices and common patterns +5. **[gotchas.md](gotchas.md)** - Critical pitfalls and troubleshooting + +## Binding Catalog + +### Storage Bindings + +| Binding | Use Case | Access Pattern | +|---------|----------|----------------| +| **KV** | Key-value cache, CDN-backed reads | `env.MY_KV.get(key)` | +| **R2** | Object storage (S3-compatible) | `env.MY_BUCKET.get(key)` | +| **D1** | SQL database (SQLite) | `env.DB.prepare(sql).all()` | +| **Durable Objects** | Coordination, real-time state | `env.MY_DO.get(id)` | +| **Vectorize** | Vector embeddings search | `env.VECTORIZE.query(vector)` | +| **Queues** | Async message processing | `env.MY_QUEUE.send(msg)` | + +### Compute Bindings + +| Binding | Use Case | Access Pattern | +|---------|----------|----------------| +| **Service** | Worker-to-Worker RPC | `env.MY_SERVICE.fetch(req)` | +| **Workers AI** | LLM inference | `env.AI.run(model, input)` | +| **Browser Rendering** | Headless Chrome | `env.BROWSER.fetch(url)` | + +### Platform Bindings + +| Binding | Use Case | Access Pattern | +|---------|----------|----------------| +| **Analytics Engine** | Custom metrics | `env.ANALYTICS.writeDataPoint(data)` | +| **mTLS** | Client certificates | `env.MY_CERT` (string) | +| **Hyperdrive** | Database pooling | `env.HYPERDRIVE.connectionString` | +| **Rate Limiting** | Request throttling | `env.RATE_LIMITER.limit(id)` | +| **Workflows** | Long-running workflows | `env.MY_WORKFLOW.create()` | + +### Configuration Bindings + +| Binding | Use Case | Access Pattern | +|---------|----------|----------------| +| **Environment Variables** | Non-sensitive config | `env.API_URL` (string) | +| **Secrets** | Sensitive values | `env.API_KEY` (string) | +| **Text/Data Blobs** | Static files | `env.MY_BLOB` (string) | +| **WASM** | WebAssembly modules | `env.MY_WASM` (WebAssembly.Module) | + +## Quick Selection Guide + +**Need persistent storage?** +- Key-value < 25MB → **KV** +- Files/objects → **R2** +- Relational data → **D1** +- Real-time coordination → **Durable Objects** + +**Need AI/compute?** +- LLM inference → **Workers AI** +- Scraping/PDFs → **Browser Rendering** +- Call another Worker → **Service binding** + +**Need async processing?** +- Background jobs → **Queues** + +**Need config?** +- Public values → **Environment Variables** +- Secrets → **Secrets** (never commit) + +## Quick Start + +1. **Add binding to wrangler.jsonc:** +```jsonc +{ + "kv_namespaces": [ + { "binding": "MY_KV", "id": "your-kv-id" } + ] +} +``` + +2. **Generate types:** +```bash +npx wrangler types +``` + +3. **Access in Worker:** +```typescript +export default { + async fetch(request, env, ctx) { + await env.MY_KV.put('key', 'value'); + return new Response('OK'); + } +} +``` + +## Type Safety + +Bindings are fully typed via `wrangler types`. See [api.md](api.md) for details. + +## Limits + +- 64 bindings max per Worker (all types combined) +- See [gotchas.md](gotchas.md) for per-binding limits + +## Key Concepts + +**Zero-overhead access:** Bindings compiled into Worker, no network calls to access +**Type-safe:** Full TypeScript support via `wrangler types` +**Per-environment:** Different IDs for dev/staging/production +**Secrets vs Vars:** Secrets encrypted at rest, never in config files + +## See Also + +- [Cloudflare Docs: Bindings](https://developers.cloudflare.com/workers/runtime-apis/bindings/) +- [Wrangler Configuration](https://developers.cloudflare.com/workers/wrangler/configuration/) diff --git a/plugins/cloudflare/skills/cloudflare/references/bindings/api.md b/plugins/cloudflare/skills/cloudflare/references/bindings/api.md new file mode 100644 index 0000000..a8ab13b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bindings/api.md @@ -0,0 +1,203 @@ +# Bindings API Reference + +## TypeScript Types + +Cloudflare generates binding types via `npx wrangler types`. This creates `.wrangler/types/runtime.d.ts` with your Env interface. + +### Generated Env Interface + +After running `wrangler types`, TypeScript knows your bindings: + +```typescript +interface Env { + // From wrangler.jsonc bindings + MY_KV: KVNamespace; + MY_BUCKET: R2Bucket; + DB: D1Database; + MY_SERVICE: Fetcher; + AI: Ai; + + // From vars + API_URL: string; + + // From secrets (set via wrangler secret put) + API_KEY: string; +} +``` + +### Binding Types + +| Config | TypeScript Type | Package | +|--------|-----------------|---------| +| `kv_namespaces` | `KVNamespace` | `@cloudflare/workers-types` | +| `r2_buckets` | `R2Bucket` | `@cloudflare/workers-types` | +| `d1_databases` | `D1Database` | `@cloudflare/workers-types` | +| `durable_objects.bindings` | `DurableObjectNamespace` | `@cloudflare/workers-types` | +| `vectorize` | `VectorizeIndex` | `@cloudflare/workers-types` | +| `queues.producers` | `Queue` | `@cloudflare/workers-types` | +| `services` | `Fetcher` | `@cloudflare/workers-types` | +| `ai` | `Ai` | `@cloudflare/workers-types` | +| `browser` | `Fetcher` | `@cloudflare/workers-types` | +| `analytics_engine_datasets` | `AnalyticsEngineDataset` | `@cloudflare/workers-types` | +| `hyperdrive` | `Hyperdrive` | `@cloudflare/workers-types` | +| `rate_limiting` | `RateLimit` | `@cloudflare/workers-types` | +| `workflows` | `Workflow` | `@cloudflare/workers-types` | +| `mtls_certificates` / `vars` / `text_blobs` / `data_blobs` | `string` | Built-in | +| `wasm_modules` | `WebAssembly.Module` | Built-in | + +## Accessing Bindings + +### Method 1: fetch() Handler (Recommended) + +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const value = await env.MY_KV.get('key'); + return new Response(value); + } +} +``` + +**Why:** Type-safe, aligns with Workers API, supports ctx for waitUntil/passThroughOnException. + +### Method 2: Hono Framework + +```typescript +import { Hono } from 'hono'; + +const app = new Hono<{ Bindings: Env }>(); + +app.get('/', async (c) => { + const value = await c.env.MY_KV.get('key'); + return c.json({ value }); +}); + +export default app; +``` + +**Why:** c.env auto-typed, ergonomic for routing-heavy apps. + +### Method 3: Module Workers (Legacy) + +```typescript +export async function handleRequest(request: Request, env: Env): Promise { + const value = await env.MY_KV.get('key'); + return new Response(value); +} + +addEventListener('fetch', (event) => { + // env not directly available - requires workarounds +}); +``` + +**Avoid:** Use fetch() handler instead (Method 1). + +## Type Generation Workflow + +### Initial Setup + +```bash +# Install wrangler +npm install -D wrangler + +# Generate types from wrangler.jsonc +npx wrangler types +``` + +### After Changing Bindings + +```bash +# Added/modified binding in wrangler.jsonc +npx wrangler types + +# TypeScript now sees updated Env interface +``` + +**Note:** `wrangler types` outputs to `.wrangler/types/runtime.d.ts`. TypeScript picks this up automatically if `@cloudflare/workers-types` is in `tsconfig.json` `"types"` array. + +## Key Binding Methods + +**KV:** +```typescript +await env.MY_KV.get(key, { type: 'json' }); // text|json|arrayBuffer|stream +await env.MY_KV.put(key, value, { expirationTtl: 3600 }); +await env.MY_KV.delete(key); +await env.MY_KV.list({ prefix: 'user:' }); +``` + +**R2:** +```typescript +await env.BUCKET.get(key); +await env.BUCKET.put(key, value); +await env.BUCKET.delete(key); +await env.BUCKET.list({ prefix: 'images/' }); +``` + +**D1:** +```typescript +await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); +await env.DB.batch([stmt1, stmt2]); +``` + +**Service:** +```typescript +await env.MY_SERVICE.fetch(new Request('https://fake/path')); +``` + +**Workers AI:** +```typescript +await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' }); +``` + +**Queues:** +```typescript +await env.MY_QUEUE.send({ userId: 123, action: 'process' }); +``` + +**Durable Objects:** +```typescript +const id = env.MY_DO.idFromName('user-123'); +const stub = env.MY_DO.get(id); +await stub.fetch(new Request('https://fake/increment')); +``` + +## Runtime vs Build-Time Types + +| Type Source | When Generated | Use Case | +|-------------|----------------|----------| +| `@cloudflare/workers-types` | npm install | Base Workers APIs (Request, Response, etc.) | +| `wrangler types` | After config change | Your specific bindings (Env interface) | + +**Install both:** +```bash +npm install -D @cloudflare/workers-types +npx wrangler types +``` + +## Type Safety Best Practices + +1. **Never use `any` for env:** +```typescript +// ❌ BAD +async fetch(request: Request, env: any) { } + +// ✅ GOOD +async fetch(request: Request, env: Env) { } +``` + +2. **Run wrangler types after config changes:** +```bash +# After editing wrangler.jsonc +npx wrangler types +``` + +3. **Check generated types match config:** +```bash +# View generated Env interface +cat .wrangler/types/runtime.d.ts +``` + +## See Also + +- [Workers Types Package](https://www.npmjs.com/package/@cloudflare/workers-types) +- [Wrangler Types Command](https://developers.cloudflare.com/workers/wrangler/commands/#types) \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/bindings/configuration.md b/plugins/cloudflare/skills/cloudflare/references/bindings/configuration.md new file mode 100644 index 0000000..9573500 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bindings/configuration.md @@ -0,0 +1,188 @@ +# Binding Configuration Reference + +## Storage Bindings + +```jsonc +{ + "kv_namespaces": [{ "binding": "MY_KV", "id": "..." }], + "r2_buckets": [{ "binding": "MY_BUCKET", "bucket_name": "my-bucket" }], + "d1_databases": [{ "binding": "DB", "database_name": "my-db", "database_id": "..." }], + "durable_objects": { "bindings": [{ "name": "MY_DO", "class_name": "MyDO" }] }, + "vectorize": [{ "binding": "VECTORIZE", "index_name": "my-index" }], + "queues": { "producers": [{ "binding": "MY_QUEUE", "queue": "my-queue" }] } +} +``` + +**Create commands:** +```bash +npx wrangler kv namespace create MY_KV +npx wrangler r2 bucket create my-bucket +npx wrangler d1 create my-db +npx wrangler vectorize create my-index --dimensions=768 --metric=cosine +npx wrangler queues create my-queue + +# List existing resources +npx wrangler kv namespace list +npx wrangler r2 bucket list +npx wrangler d1 list +npx wrangler vectorize list +npx wrangler queues list +``` + +## Compute Bindings + +```jsonc +{ + "services": [{ + "binding": "MY_SERVICE", + "service": "other-worker", + "environment": "production" // Optional: target specific env + }], + "ai": { "binding": "AI" }, + "browser": { "binding": "BROWSER" }, + "workflows": [{ "binding": "MY_WORKFLOW", "name": "my-workflow" }] +} +``` + +**Create workflows:** +```bash +npx wrangler workflows create my-workflow +``` + +## Platform Bindings + +```jsonc +{ + "analytics_engine_datasets": [{ "binding": "ANALYTICS" }], + "mtls_certificates": [{ "binding": "MY_CERT", "certificate_id": "..." }], + "hyperdrive": [{ "binding": "HYPERDRIVE", "id": "..." }], + "unsafe": { + "bindings": [{ "name": "RATE_LIMITER", "type": "ratelimit", "namespace_id": "..." }] + } +} +``` + +## Configuration Bindings + +```jsonc +{ + "vars": { + "API_URL": "https://api.example.com", + "MAX_RETRIES": "3" + }, + "text_blobs": { "MY_TEXT": "./data/template.html" }, + "data_blobs": { "MY_DATA": "./data/config.bin" }, + "wasm_modules": { "MY_WASM": "./build/module.wasm" } +} +``` + +**Secrets (never in config):** +```bash +npx wrangler secret put API_KEY +``` + +## Environment-Specific Configuration + +```jsonc +{ + "name": "my-worker", + "vars": { "ENV": "production" }, + "kv_namespaces": [{ "binding": "CACHE", "id": "prod-kv-id" }], + + "env": { + "staging": { + "vars": { "ENV": "staging" }, + "kv_namespaces": [{ "binding": "CACHE", "id": "staging-kv-id" }] + } + } +} +``` + +**Deploy:** +```bash +npx wrangler deploy # Production +npx wrangler deploy --env staging +``` + +## Local Development + +```jsonc +{ + "kv_namespaces": [{ + "binding": "MY_KV", + "id": "prod-id", + "preview_id": "dev-id" // Used in wrangler dev + }] +} +``` + +**Or use remote:** +```bash +npx wrangler dev --remote # Uses production bindings +``` + +## Complete Example + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-app", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", + + "vars": { "API_URL": "https://api.example.com" }, + "kv_namespaces": [{ "binding": "CACHE", "id": "abc123" }], + "r2_buckets": [{ "binding": "ASSETS", "bucket_name": "my-assets" }], + "d1_databases": [{ "binding": "DB", "database_name": "my-db", "database_id": "xyz789" }], + "services": [{ "binding": "AUTH", "service": "auth-worker" }], + "ai": { "binding": "AI" } +} +``` + +## Binding-Specific Configuration + +### Durable Objects with Class Export + +```jsonc +{ + "durable_objects": { + "bindings": [ + { "name": "COUNTER", "class_name": "Counter", "script_name": "my-worker" } + ] + } +} +``` + +```typescript +// In same Worker or script_name Worker +export class Counter { + constructor(private state: DurableObjectState, private env: Env) {} + async fetch(request: Request) { /* ... */ } +} +``` + +### Queue Consumers + +```jsonc +{ + "queues": { + "producers": [{ "binding": "MY_QUEUE", "queue": "my-queue" }], + "consumers": [{ "queue": "my-queue", "max_batch_size": 10 }] + } +} +``` + +Queue consumer handler: `export default { async queue(batch, env) { /* process batch.messages */ } }` + +## Key Points + +- **64 binding limit** (all types combined) +- **Secrets**: Always use `wrangler secret put`, never commit +- **Types**: Run `npx wrangler types` after config changes +- **Environments**: Use `env` field for staging/production variants +- **Development**: Use `preview_id` or `--remote` flag +- **IDs vs Names**: Some bindings use `id` (KV, D1), others use `name` (R2, Queues) + +## See Also + +- [Wrangler Configuration](https://developers.cloudflare.com/workers/wrangler/configuration/) \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/bindings/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/bindings/gotchas.md new file mode 100644 index 0000000..90aca1a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bindings/gotchas.md @@ -0,0 +1,208 @@ +# Binding Gotchas and Troubleshooting + +## Critical: Global Scope Mutation + +### ❌ THE #1 GOTCHA: Caching env in Global Scope + +```typescript +// ❌ DANGEROUS - env cached at deploy time +const apiKey = env.API_KEY; // ERROR: env not available in global scope + +export default { + async fetch(request: Request, env: Env) { + // Uses undefined or stale value! + } +} +``` + +**Why it breaks:** +- `env` not available in global scope +- If using workarounds, secrets may not update without redeployment +- Leads to "Cannot read property 'X' of undefined" errors + +**✅ Always access env per-request:** +```typescript +export default { + async fetch(request: Request, env: Env) { + const apiKey = env.API_KEY; // Fresh every request + } +} +``` + +## Common Errors + +### "env.MY_KV is undefined" + +**Cause:** Name mismatch or not configured +**Solution:** Check wrangler.jsonc (case-sensitive), run `npx wrangler types`, verify `npx wrangler kv namespace list` + +### "Property 'MY_KV' does not exist on type 'Env'" + +**Cause:** Types not generated +**Solution:** `npx wrangler types` + +### "preview_id is required for --remote" + +**Cause:** Missing preview binding +**Solution:** Add `"preview_id": "dev-id"` or use `npx wrangler dev` (local mode) + +### "Secret updated but Worker still uses old value" + +**Cause:** Cached in global scope or not redeployed +**Solution:** Avoid global caching, redeploy after secret change + +### "KV get() returns null for existing key" + +**Cause:** Eventual consistency (60s), wrong namespace, wrong environment +**Solution:** +```bash +# Check key exists +npx wrangler kv key get --binding=MY_KV "your-key" + +# Verify namespace ID +npx wrangler kv namespace list + +# Check environment +npx wrangler deployments list +``` + +### "D1 database not found" + +**Solution:** `npx wrangler d1 list`, verify ID in wrangler.jsonc + +### "Service binding returns 'No such service'" + +**Cause:** Target Worker not deployed, name mismatch, environment mismatch +**Solution:** +```bash +# List deployed Workers +npx wrangler deployments list --name=target-worker + +# Check service binding config +cat wrangler.jsonc | grep -A2 services + +# Deploy target first +cd ../target-worker && npx wrangler deploy +``` + +### "Rate limit exceeded" on KV writes + +**Cause:** >1 write/second per key +**Solution:** Use different keys, Durable Objects, or Queues + +## Type Safety Gotchas + +### Missing @cloudflare/workers-types + +**Error:** `Cannot find name 'Request'` +**Solution:** `npm install -D @cloudflare/workers-types`, add to tsconfig.json `"types"` + +### Binding Type Mismatches + +```typescript +// ❌ Wrong - KV returns string | null +const value: string = await env.MY_KV.get('key'); + +// ✅ Handle null +const value = await env.MY_KV.get('key'); +if (!value) return new Response('Not found', { status: 404 }); +``` + +## Environment Gotchas + +### Wrong Environment Deployed + +**Solution:** Check `npx wrangler deployments list`, use `--env` flag + +### Secrets Not Per-Environment + +**Solution:** Set per environment: `npx wrangler secret put API_KEY --env staging` + +## Development Gotchas + +**wrangler dev vs deploy:** +- dev: Uses `preview_id` or local bindings, secrets not available +- deploy: Uses production `id`, secrets available + +**Access secrets in dev:** `npx wrangler dev --remote` +**Persist local data:** `npx wrangler dev --persist` + +## Performance Gotchas + +### Sequential Binding Calls + +```typescript +// ❌ Slow +const user = await env.DB.prepare('...').first(); +const config = await env.MY_KV.get('config'); + +// ✅ Parallel +const [user, config] = await Promise.all([ + env.DB.prepare('...').first(), + env.MY_KV.get('config') +]); +``` + +## Security Gotchas + +**❌ Secrets in logs:** `console.log('Key:', env.API_KEY)` - visible in dashboard +**✅** `console.log('Key:', env.API_KEY ? '***' : 'missing')` + +**❌ Exposing env:** `return Response.json(env)` - exposes all bindings +**✅** Never return env object in responses + +## Limits Reference + +| Resource | Limit | Impact | Plan | +|----------|-------|--------|------| +| **Bindings per Worker** | 64 total | All binding types combined | All | +| **Environment variables** | 64 max, 5KB each | Per Worker | All | +| **Secret size** | 1KB | Per secret | All | +| **KV key size** | 512 bytes | UTF-8 encoded | All | +| **KV value size** | 25 MB | Per value | All | +| **KV writes per key** | 1/second | Per key; exceeding = 429 error | All | +| **KV list() results** | 1000 keys | Per call; use cursor for more | All | +| **KV operations** | 1000 reads/day | Free tier only | Free | +| **R2 object size** | 5 TB | Per object | All | +| **R2 operations** | 1M Class A/month free | Writes | All | +| **D1 database size** | 10 GB | Per database | All | +| **D1 rows per query** | 100,000 | Result set limit | All | +| **D1 databases** | 10 | Free tier | Free | +| **Queue batch size** | 100 messages | Per consumer batch | All | +| **Queue message size** | 128 KB | Per message | All | +| **Service binding calls** | Unlimited | Counts toward CPU time | All | +| **Durable Objects** | 1M requests/month free | First 1M | Free | + +## Debugging Tips + +```bash +# Check configuration +npx wrangler deploy --dry-run # Validate config without deploying +npx wrangler kv namespace list # List KV namespaces +npx wrangler secret list # List secrets (not values) +npx wrangler deployments list # Recent deployments + +# Inspect bindings +npx wrangler kv key list --binding=MY_KV +npx wrangler kv key get --binding=MY_KV "key-name" +npx wrangler r2 object get my-bucket/file.txt +npx wrangler d1 execute my-db --command="SELECT * FROM sqlite_master" + +# Test locally +npx wrangler dev # Local mode +npx wrangler dev --remote # Production bindings +npx wrangler dev --persist # Persist data across restarts + +# Verify types +npx wrangler types +cat .wrangler/types/runtime.d.ts | grep "interface Env" + +# Debug specific binding issues +npx wrangler tail # Stream logs in real-time +npx wrangler tail --format=pretty # Formatted logs +``` + +## See Also + +- [Workers Limits](https://developers.cloudflare.com/workers/platform/limits/) +- [Wrangler Commands](https://developers.cloudflare.com/workers/wrangler/commands/) diff --git a/plugins/cloudflare/skills/cloudflare/references/bindings/patterns.md b/plugins/cloudflare/skills/cloudflare/references/bindings/patterns.md new file mode 100644 index 0000000..759305d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bindings/patterns.md @@ -0,0 +1,200 @@ +# Binding Patterns and Best Practices + +## Service Binding Patterns + +### RPC via Service Bindings + +```typescript +// auth-worker +export default { + async fetch(request: Request, env: Env) { + const token = request.headers.get('Authorization'); + return new Response(JSON.stringify({ valid: await validateToken(token) })); + } +} + +// api-worker +const response = await env.AUTH_SERVICE.fetch( + new Request('https://fake-host/validate', { + headers: { 'Authorization': token } + }) +); +``` + +**Why RPC?** Zero latency (same datacenter), no DNS, free, type-safe. + +**HTTP vs Service:** +```typescript +// ❌ HTTP (slow, paid, cross-region latency) +await fetch('https://auth-worker.example.com/validate'); + +// ✅ Service binding (fast, free, same isolate) +await env.AUTH_SERVICE.fetch(new Request('https://fake-host/validate')); +``` + +**URL doesn't matter:** Service bindings ignore hostname/protocol, routing happens via binding name. + +### Typed Service RPC + +```typescript +// shared-types.ts +export interface AuthRequest { token: string; } +export interface AuthResponse { valid: boolean; userId?: string; } + +// auth-worker +export default { + async fetch(request: Request): Promise { + const body: AuthRequest = await request.json(); + const response: AuthResponse = { valid: true, userId: '123' }; + return Response.json(response); + } +} + +// api-worker +const response = await env.AUTH_SERVICE.fetch( + new Request('https://fake/validate', { + method: 'POST', + body: JSON.stringify({ token } satisfies AuthRequest) + }) +); +const data: AuthResponse = await response.json(); +``` + +## Secrets Management + +```bash +# Set secret +npx wrangler secret put API_KEY +cat api-key.txt | npx wrangler secret put API_KEY +npx wrangler secret put API_KEY --env staging +``` + +```typescript +// Use secret +const response = await fetch('https://api.example.com', { + headers: { 'Authorization': `Bearer ${env.API_KEY}` } +}); +``` + +**Never commit secrets:** +```jsonc +// ❌ NEVER +{ "vars": { "API_KEY": "sk_live_abc123" } } +``` + +## Testing with Mock Bindings + +### Vitest Mock + +```typescript +import { vi } from 'vitest'; + +const mockKV: KVNamespace = { + get: vi.fn(async (key) => key === 'test' ? 'value' : null), + put: vi.fn(async () => {}), + delete: vi.fn(async () => {}), + list: vi.fn(async () => ({ keys: [], list_complete: true, cursor: '' })), + getWithMetadata: vi.fn(), +} as unknown as KVNamespace; + +const mockEnv: Env = { MY_KV: mockKV }; +const mockCtx: ExecutionContext = { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), +}; + +const response = await worker.fetch( + new Request('http://localhost/test'), + mockEnv, + mockCtx +); +``` + +## Binding Access Patterns + +### Lazy Access + +```typescript +// ✅ Access only when needed +if (url.pathname === '/cached') { + const cached = await env.MY_KV.get('data'); + if (cached) return new Response(cached); +} +``` + +### Parallel Access + +```typescript +// ✅ Parallelize independent calls +const [user, config, cache] = await Promise.all([ + env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(), + env.MY_KV.get('config'), + env.CACHE.get('data') +]); +``` + +## Storage Selection + +### KV: CDN-Backed Reads + +```typescript +const config = await env.MY_KV.get('app-config', { type: 'json' }); +``` + +**Use when:** Read-heavy, <25MB, global distribution, eventual consistency OK +**Latency:** <10ms reads (cached), writes eventually consistent (60s) + +### D1: Relational Queries + +```typescript +const results = await env.DB.prepare(` + SELECT u.name, COUNT(o.id) FROM users u + LEFT JOIN orders o ON u.id = o.user_id GROUP BY u.id +`).all(); +``` + +**Use when:** Relational data, JOINs, ACID transactions +**Limits:** 10GB database size, 100k rows per query + +### R2: Large Objects + +```typescript +const object = await env.MY_BUCKET.get('large-file.zip'); +return new Response(object.body); +``` + +**Use when:** Files >25MB, S3-compatible API needed +**Limits:** 5TB per object, unlimited storage + +### Durable Objects: Coordination + +```typescript +const id = env.COUNTER.idFromName('global'); +const stub = env.COUNTER.get(id); +await stub.fetch(new Request('https://fake/increment')); +``` + +**Use when:** Strong consistency, real-time coordination, WebSocket state +**Guarantees:** Single-threaded execution, transactional storage + +## Anti-Patterns + +**❌ Hardcoding credentials:** `const apiKey = 'sk_live_abc123'` +**✅** `npx wrangler secret put API_KEY` + +**❌ Using REST API:** `fetch('https://api.cloudflare.com/.../kv/...')` +**✅** `env.MY_KV.get('key')` + +**❌ Polling storage:** `setInterval(() => env.KV.get('config'), 1000)` +**✅** Use Durable Objects for real-time state + +**❌ Large data in vars:** `{ "vars": { "HUGE_CONFIG": "..." } }` (5KB max) +**✅** `env.MY_KV.put('config', data)` + +**❌ Caching env globally:** `const apiKey = env.API_KEY` outside fetch() +**✅** Access `env.API_KEY` per-request inside fetch() + +## See Also + +- [Service Bindings Docs](https://developers.cloudflare.com/workers/runtime-apis/bindings/service-bindings/) +- [Miniflare Testing](https://miniflare.dev/) \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/bot-management/README.md b/plugins/cloudflare/skills/cloudflare/references/bot-management/README.md new file mode 100644 index 0000000..a76808d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bot-management/README.md @@ -0,0 +1,94 @@ +# Cloudflare Bot Management + +Enterprise-grade bot detection, protection, and mitigation using ML/heuristics, bot scores, JavaScript detections, and verified bot handling. + +## Overview + +Bot Management provides multi-tier protection: +- **Free (Bot Fight Mode)**: Auto-blocks definite bots, no config +- **Pro/Business (Super Bot Fight Mode)**: Configurable actions, static resource protection, analytics groupings +- **Enterprise (Bot Management)**: Granular 1-99 scores, WAF integration, JA3/JA4 fingerprinting, Workers API, Advanced Analytics + +## Quick Start + +```txt +# Dashboard: Security > Bots +# Enterprise: Deploy rule template +(cf.bot_management.score eq 1 and not cf.bot_management.verified_bot) → Block +(cf.bot_management.score le 29 and not cf.bot_management.verified_bot) → Managed Challenge +``` + +## What Do You Need? + +```txt +├─ Initial setup → configuration.md +│ ├─ Free tier → "Bot Fight Mode" +│ ├─ Pro/Business → "Super Bot Fight Mode" +│ └─ Enterprise → "Bot Management for Enterprise" +├─ Workers API integration → api.md +├─ WAF rules → patterns.md +├─ Debugging → gotchas.md +└─ Analytics → api.md#bot-analytics +``` + +## Reading Order + +| Task | Files to Read | +|------|---------------| +| Enable bot protection | README → configuration.md | +| Workers bot detection | README → api.md | +| WAF rule templates | README → patterns.md | +| Debug bot issues | gotchas.md | +| Advanced analytics | api.md#bot-analytics | + +## Core Concepts + +**Bot Scores**: 1-99 (1 = definitely automated, 99 = definitely human). Threshold: <30 indicates bot traffic. Enterprise gets granular 1-99; Pro/Business get groupings only. + +**Detection Engines**: Heuristics (known fingerprints, assigns score=1), ML (majority of detections, supervised learning on billions of requests), Anomaly Detection (optional, baseline traffic analysis), JavaScript Detections (headless browser detection). + +**Verified Bots**: Allowlisted good bots (search engines, AI crawlers) verified via reverse DNS or Web Bot Auth. Access via `cf.bot_management.verified_bot` or `cf.verified_bot_category`. + +## Platform Limits + +| Plan | Bot Scores | JA3/JA4 | Custom Rules | Analytics Retention | +|------|------------|---------|--------------|---------------------| +| Free | No (auto-block only) | No | 5 | N/A (no analytics) | +| Pro/Business | Groupings only | No | 20/100 | 30 days (72h at a time) | +| Enterprise | 1-99 granular | Yes | 1,000+ | 30 days (1 week at a time) | + +## Basic Patterns + +```typescript +// Workers: Check bot score +export default { + async fetch(request: Request): Promise { + const botScore = request.cf?.botManagement?.score; + if (botScore && botScore < 30 && !request.cf?.botManagement?.verifiedBot) { + return new Response('Bot detected', { status: 403 }); + } + return fetch(request); + } +}; +``` + +```txt +# WAF: Block definite bots +(cf.bot_management.score eq 1 and not cf.bot_management.verified_bot) + +# WAF: Protect sensitive endpoints +(cf.bot_management.score lt 50 and http.request.uri.path in {"/login" "/checkout"} and not cf.bot_management.verified_bot) +``` + +## In This Reference + +- [configuration.md](./configuration.md) - Product tiers, WAF rule setup, JavaScript Detections, ML auto-updates +- [api.md](./api.md) - Workers BotManagement interface, WAF fields, JA4 Signals +- [patterns.md](./patterns.md) - E-commerce, API protection, mobile app allowlisting, SEO-friendly handling +- [gotchas.md](./gotchas.md) - False positives/negatives, score=0 issues, JSD limitations, CSP requirements + +## See Also + +- [waf](../waf/) - WAF custom rules for bot enforcement +- [workers](../workers/) - Workers request.cf.botManagement API +- [api-shield](../api-shield/) - API-specific bot protection diff --git a/plugins/cloudflare/skills/cloudflare/references/bot-management/api.md b/plugins/cloudflare/skills/cloudflare/references/bot-management/api.md new file mode 100644 index 0000000..eda958a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bot-management/api.md @@ -0,0 +1,169 @@ +# Bot Management API + +## Workers: BotManagement Interface + +```typescript +interface BotManagement { + score: number; // 1-99 (Enterprise), 0 if not computed + verifiedBot: boolean; // Is verified bot + staticResource: boolean; // Serves static resource + ja3Hash: string; // JA3 fingerprint (Enterprise, HTTPS only) + ja4: string; // JA4 fingerprint (Enterprise, HTTPS only) + jsDetection?: { + passed: boolean; // Passed JS detection (if enabled) + }; + detectionIds: number[]; // Heuristic detection IDs + corporateProxy?: boolean; // From corporate proxy (Enterprise) +} + +// DEPRECATED: Use botManagement.score instead +// request.cf.clientTrustScore (legacy, duplicate of botManagement.score) + +// Access via request.cf +import type { IncomingRequestCfProperties } from '@cloudflare/workers-types'; + +export default { + async fetch(request: Request): Promise { + const cf = request.cf as IncomingRequestCfProperties | undefined; + const botMgmt = cf?.botManagement; + + if (!botMgmt) return fetch(request); + if (botMgmt.verifiedBot) return fetch(request); // Allow verified bots + if (botMgmt.score === 1) return new Response('Blocked', { status: 403 }); + if (botMgmt.score < 30) return new Response('Challenge required', { status: 429 }); + + return fetch(request); + } +}; +``` + +## WAF Fields Reference + +```txt +# Score fields +cf.bot_management.score # 0-99 (0 = not computed) +cf.bot_management.verified_bot # boolean +cf.bot_management.static_resource # boolean +cf.bot_management.ja3_hash # string (Enterprise) +cf.bot_management.ja4 # string (Enterprise) +cf.bot_management.detection_ids # array +cf.bot_management.js_detection.passed # boolean +cf.bot_management.corporate_proxy # boolean (Enterprise) +cf.verified_bot_category # string + +# Workers equivalent +request.cf.botManagement.score +request.cf.botManagement.verifiedBot +request.cf.botManagement.ja3Hash +request.cf.botManagement.ja4 +request.cf.botManagement.jsDetection.passed +request.cf.verifiedBotCategory +``` + +## JA4 Signals (Enterprise) + +```typescript +import type { IncomingRequestCfProperties } from '@cloudflare/workers-types'; + +interface JA4Signals { + // Ratios (0.0-1.0) + heuristic_ratio_1h?: number; // Fraction flagged by heuristics + browser_ratio_1h?: number; // Fraction from real browsers + cache_ratio_1h?: number; // Fraction hitting cache + h2h3_ratio_1h?: number; // Fraction using HTTP/2 or HTTP/3 + // Ranks (relative position in distribution) + uas_rank_1h?: number; // User-Agent diversity rank + paths_rank_1h?: number; // Path diversity rank + reqs_rank_1h?: number; // Request volume rank + ips_rank_1h?: number; // IP diversity rank + // Quantiles (0.0-1.0, percentile in distribution) + reqs_quantile_1h?: number; // Request volume quantile + ips_quantile_1h?: number; // IP count quantile +} + +export default { + async fetch(request: Request): Promise { + const cf = request.cf as IncomingRequestCfProperties | undefined; + const ja4Signals = cf?.ja4Signals as JA4Signals | undefined; + + if (!ja4Signals) return fetch(request); // Not available for HTTP or Worker routing + + // Check for anomalous behavior + // High heuristic_ratio or low browser_ratio = suspicious + const heuristicRatio = ja4Signals.heuristic_ratio_1h ?? 0; + const browserRatio = ja4Signals.browser_ratio_1h ?? 0; + + if (heuristicRatio > 0.5 || browserRatio < 0.3) { + return new Response('Suspicious traffic', { status: 403 }); + } + + return fetch(request); + } +}; +``` + +## Common Patterns + +See [patterns.md](./patterns.md) for Workers examples: mobile app allowlisting, corporate proxy exemption, datacenter detection, conditional delay, and more. + +## Bot Analytics + +### Access Locations +- Dashboard: Security > Bots (old) or Security > Analytics > Bot analysis (new) +- GraphQL API for programmatic access +- Security Events & Security Analytics +- Logpush/Logpull + +### Available Data +- **Enterprise BM**: Bot scores (1-99), bot score source, distribution +- **Pro/Business**: Bot groupings (automated, likely automated, likely human) +- Top attributes: IPs, paths, user agents, countries +- Detection sources: Heuristics, ML, AD, JSD +- Verified bot categories + +### Time Ranges +- **Enterprise BM**: Up to 1 week at a time, 30 days history +- **Pro/Business**: Up to 72 hours at a time, 30 days history +- Real-time in most cases, adaptive sampling (1-10% depending on volume) + +## Logpush Fields + +```txt +BotScore # 1-99 or 0 if not computed +BotScoreSrc # Detection engine (ML, Heuristics, etc.) +BotTags # Classification tags +BotDetectionIDs # Heuristic detection IDs +``` + +**BotScoreSrc values:** +- `"Heuristics"` - Known fingerprint +- `"Machine Learning"` - ML model +- `"Anomaly Detection"` - Baseline anomaly +- `"JS Detection"` - JavaScript check +- `"Cloudflare Service"` - Zero Trust +- `"Not Computed"` - Score = 0 + +Access via Logpush (stream to cloud storage/SIEM), Logpull (API to fetch logs), or GraphQL API (query analytics data). + +## Testing with Miniflare + +Miniflare provides mock botManagement data for local development: + +**Default values:** +- `score: 99` (human) +- `verifiedBot: false` +- `corporateProxy: false` +- `ja3Hash: "25b4882c2bcb50cd6b469ff28c596742"` +- `staticResource: false` +- `detectionIds: []` + +**Override in tests:** +```typescript +import { getPlatformProxy } from 'wrangler'; + +const { cf, dispose } = await getPlatformProxy(); +// cf.botManagement is frozen mock object +expect(cf.botManagement.score).toBe(99); +``` + +For custom test data, mock request.cf in your test setup. diff --git a/plugins/cloudflare/skills/cloudflare/references/bot-management/configuration.md b/plugins/cloudflare/skills/cloudflare/references/bot-management/configuration.md new file mode 100644 index 0000000..4166bcc --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bot-management/configuration.md @@ -0,0 +1,163 @@ +# Bot Management Configuration + +## Product Tiers + +**Note:** Dashboard paths differ between old and new UI: +- **New:** Security > Settings > Filter "Bot traffic" +- **Old:** Security > Bots + +Both UIs access same settings. + +### Bot Score Groupings (Pro/Business) + +Pro/Business users see bot score groupings instead of granular 1-99 scores: + +| Score | Grouping | Meaning | +|-------|----------|---------| +| 0 | Not computed | Bot Management didn't run | +| 1 | Automated | Definite bot (heuristic match) | +| 2-29 | Likely automated | Probably bot (ML detection) | +| 30-99 | Likely human | Probably human | +| N/A | Verified bot | Allowlisted good bot | + +Enterprise plans get granular 1-99 scores for custom thresholds. + +### Bot Fight Mode (Free) +- Auto-blocks definite bots (score=1), excludes verified bots by default +- JavaScript Detections always enabled, no configuration options + +### Super Bot Fight Mode (Pro/Business) +```txt +Dashboard: Security > Bots > Configure +- Definitely automated: Block/Challenge +- Likely automated: Challenge/Allow +- Verified bots: Allow (recommended) +- Static resource protection: ON (may block mail clients) +- JavaScript Detections: Optional +``` + +### Bot Management for Enterprise +```txt +Dashboard: Security > Bots > Configure > Auto-updates: ON (recommended) + +# Template 1: Block definite bots +(cf.bot_management.score eq 1 and not cf.bot_management.verified_bot and not cf.bot_management.static_resource) +Action: Block + +# Template 2: Challenge likely bots +(cf.bot_management.score ge 2 and cf.bot_management.score le 29 and not cf.bot_management.verified_bot and not cf.bot_management.static_resource) +Action: Managed Challenge +``` + +## JavaScript Detections Setup + +### Enable via Dashboard +```txt +Security > Bots > Configure Bot Management > JS Detections: ON + +Update CSP: script-src 'self' /cdn-cgi/challenge-platform/; +``` + +### Manual JS Injection (API) +```html + + +``` + +**Use API for**: Selective deployment on specific pages +**Don't combine**: Zone-wide toggle + manual injection + +### WAF Rules for JSD +```txt +# NEVER use on first page visit (needs HTML page first) +(not cf.bot_management.js_detection.passed and http.request.uri.path eq "/api/user/create" and http.request.method eq "POST" and not cf.bot_management.verified_bot) +Action: Managed Challenge (always use Managed Challenge, not Block) +``` + +### Limitations +- First request won't have JSD data (needs HTML page first) +- Strips ETags from HTML responses +- Not supported with CSP via `` tags +- Websocket endpoints not supported +- Native mobile apps won't pass +- cf_clearance cookie: 15-minute lifespan, max 4096 bytes + +## __cf_bm Cookie + +Cloudflare sets `__cf_bm` cookie to smooth bot scores across user sessions: + +- **Purpose:** Reduces false positives from score volatility +- **Scope:** Per-domain, HTTP-only +- **Lifespan:** Session duration +- **Privacy:** No PII—only session classification +- **Automatic:** No configuration required + +Bot scores for repeat visitors consider session history via this cookie. + +## Static Resource Protection + +**File Extensions**: ico, jpg, png, jpeg, gif, css, js, tif, tiff, bmp, pict, webp, svg, svgz, class, jar, txt, csv, doc, docx, xls, xlsx, pdf, ps, pls, ppt, pptx, ttf, otf, woff, woff2, eot, eps, ejs, swf, torrent, midi, mid, m3u8, m4a, mp3, ogg, ts +**Plus**: `/.well-known/` path (all files) + +```txt +# Exclude static resources from bot rules +(cf.bot_management.score lt 30 and not cf.bot_management.static_resource) +``` + +**WARNING**: May block mail clients fetching static images + +## JA3/JA4 Fingerprinting (Enterprise) + +```txt +# Block specific attack fingerprint +(cf.bot_management.ja3_hash eq "8b8e3d5e3e8b3d5e") + +# Allow mobile app by fingerprint +(cf.bot_management.ja4 eq "your_mobile_app_fingerprint") +``` + +Only available for HTTPS/TLS traffic. Missing for Worker-routed traffic or HTTP requests. + +## Verified Bot Categories + +```txt +# Allow search engines only +(cf.verified_bot_category eq "Search Engine Crawler") + +# Block AI crawlers +(cf.verified_bot_category eq "AI Crawler") +Action: Block + +# Or use dashboard: Security > Settings > Bot Management > Block AI Bots +``` + +| Category | String Value | Example | +|----------|--------------|---------| +| AI Crawler | `AI Crawler` | GPTBot, Claude-Web | +| AI Assistant | `AI Assistant` | Perplexity-User, DuckAssistBot | +| AI Search | `AI Search` | OAI-SearchBot | +| Accessibility | `Accessibility` | Accessible Web Bot | +| Academic Research | `Academic Research` | Library of Congress | +| Advertising & Marketing | `Advertising & Marketing` | Google Adsbot | +| Aggregator | `Aggregator` | Pinterest, Indeed | +| Archiver | `Archiver` | Internet Archive, CommonCrawl | +| Feed Fetcher | `Feed Fetcher` | RSS/Podcast updaters | +| Monitoring & Analytics | `Monitoring & Analytics` | Uptime monitors | +| Page Preview | `Page Preview` | Facebook/Slack link preview | +| SEO | `Search Engine Optimization` | Google Lighthouse | +| Security | `Security` | Vulnerability scanners | +| Social Media Marketing | `Social Media Marketing` | Brandwatch | +| Webhooks | `Webhooks` | Payment processors | +| Other | `Other` | Uncategorized bots | + +## Best Practices + +- **ML Auto-Updates**: Enable on Enterprise for latest models +- **Start with Managed Challenge**: Test before blocking +- **Always exclude verified bots**: Use `not cf.bot_management.verified_bot` +- **Exempt corporate proxies**: For B2B traffic via `cf.bot_management.corporate_proxy` +- **Use static resource exception**: Improves performance, reduces overhead diff --git a/plugins/cloudflare/skills/cloudflare/references/bot-management/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/bot-management/gotchas.md new file mode 100644 index 0000000..685bcbd --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bot-management/gotchas.md @@ -0,0 +1,114 @@ +# Bot Management Gotchas + +## Common Errors + +### "Bot Score = 0" + +**Cause:** Bot Management didn't run (internal Cloudflare request, Worker routing to zone (Orange-to-Orange), or request handled before BM (Redirect Rules, etc.)) +**Solution:** Check request flow and ensure Bot Management runs in request lifecycle + +### "JavaScript Detections Not Working" + +**Cause:** `js_detection.passed` always false or undefined due to: CSP headers don't allow `/cdn-cgi/challenge-platform/`, using on first page visit (needs HTML page first), ad blockers or disabled JS, JSD not enabled in dashboard, or using Block action (must use Managed Challenge) +**Solution:** Add CSP header `Content-Security-Policy: script-src 'self' /cdn-cgi/challenge-platform/;` and ensure JSD is enabled with Managed Challenge action + +### "False Positives (Legitimate Users Blocked)" + +**Cause:** Bot detection incorrectly flagging legitimate users +**Solution:** Check Bot Analytics for affected IPs/paths, identify detection source (ML, Heuristics, etc.), create exception rule like `(cf.bot_management.score lt 30 and http.request.uri.path eq "/problematic-path")` with Action: Skip (Bot Management), or allowlist by IP/ASN/country + +### "False Negatives (Bots Not Caught)" + +**Cause:** Bots bypassing detection +**Solution:** Lower score threshold (30 → 50), enable JavaScript Detections, add JA3/JA4 fingerprinting rules, or use rate limiting as fallback + +### "Verified Bot Blocked" + +**Cause:** Search engine bot blocked by WAF Managed Rules (not just Bot Management) +**Solution:** Create WAF exception for specific rule ID and verify bot via reverse DNS + +### "Yandex Bot Blocked During IP Update" + +**Cause:** Yandex updates bot IPs; new IPs unrecognized for 48h during propagation +**Solution:** +1. Check Security Events for specific WAF rule ID blocking Yandex +2. Create WAF exception: + ```txt + (http.user_agent contains "YandexBot" and ip.src in {}) + Action: Skip (WAF Managed Ruleset) + ``` +3. Monitor Bot Analytics for 48h +4. Remove exception after propagation completes + +Issue resolves automatically after 48h. Contact Cloudflare Support if persists. + +### "JA3/JA4 Missing" + +**Cause:** Non-HTTPS traffic, Worker routing traffic, Orange-to-Orange traffic via Worker, or Bot Management skipped +**Solution:** JA3/JA4 only available for HTTPS/TLS traffic; check request routing + +**JA3/JA4 Not User-Unique:** Same browser/library version = same fingerprint +- Don't use for user identification +- Use for client profiling only +- Fingerprints change with browser updates + +## Bot Verification Methods + +Cloudflare verifies bots via: + +1. **Reverse DNS (IP validation):** Traditional method—bot IP resolves to expected domain +2. **Web Bot Auth:** Modern cryptographic verification—faster propagation + +When `verifiedBot=true`, bot passed at least one method. + +**Inactive verified bots:** IPs removed after 24h of no traffic. + +## Detection Engine Behavior + +| Engine | Score | Timing | Plan | Notes | +|--------|-------|--------|------|-------| +| Heuristics | Always 1 | Immediate | All | Known fingerprints—overrides ML | +| ML | 1-99 | Immediate | All | Majority of detections | +| Anomaly Detection | Influences | After baseline | Enterprise | Optional, baseline analysis | +| JavaScript Detections | Pass/fail | After JS | Pro+ | Headless browser detection | +| Cloudflare Service | N/A | N/A | Enterprise | Zero Trust internal source | + +**Priority:** Heuristics > ML—if heuristic matches, score=1 regardless of ML. + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Bot Score = 0 | Means not computed | Not score = 100 | +| First request JSD data | May not be available | JSD data appears on subsequent requests | +| Score accuracy | Not 100% guaranteed | False positives/negatives possible | +| JSD on first HTML page visit | Not supported | Requires subsequent page load | +| JSD requirements | JavaScript-enabled browser | Won't work with JS disabled or ad blockers | +| JSD ETag stripping | Strips ETags from HTML responses | May affect caching behavior | +| JSD CSP compatibility | Requires specific CSP | Not compatible with some CSP configurations | +| JSD meta CSP tags | Not supported | Must use HTTP headers | +| JSD WebSocket support | Not supported | WebSocket endpoints won't work with JSD | +| JSD mobile app support | Native apps won't pass | Only works in browsers | +| JA3/JA4 traffic type | HTTPS/TLS only | Not available for non-HTTPS traffic | +| JA3/JA4 Worker routing | Missing for Worker-routed traffic | Check request routing | +| JA3/JA4 uniqueness | Not unique per user | Shared by clients with same browser/library | +| JA3/JA4 stability | Can change with updates | Browser/library updates affect fingerprints | +| WAF custom rules (Free) | 5 | Varies by plan | +| WAF custom rules (Pro) | 20 | Varies by plan | +| WAF custom rules (Business) | 100 | Varies by plan | +| WAF custom rules (Enterprise) | 1,000+ | Varies by plan | +| Workers CPU time | Varies by plan | Applies to bot logic | +| Bot Analytics sampling | 1-10% adaptive | High-volume zones sampled more aggressively | +| Bot Analytics history | 30 days max | Historical data retention limit | +| CSP requirements for JSD | Must allow `/cdn-cgi/challenge-platform/` | Required for JSD to function | + +### Plan Restrictions + +| Feature | Free | Pro/Business | Enterprise | +|---------|------|--------------|------------| +| Granular scores (1-99) | No | No | Yes | +| JA3/JA4 | No | No | Yes | +| Anomaly Detection | No | No | Yes | +| Corporate Proxy detection | No | No | Yes | +| Verified bot categories | Limited | Limited | Full | +| Custom WAF rules | 5 | 20/100 | 1,000+ | diff --git a/plugins/cloudflare/skills/cloudflare/references/bot-management/patterns.md b/plugins/cloudflare/skills/cloudflare/references/bot-management/patterns.md new file mode 100644 index 0000000..4ca7085 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/bot-management/patterns.md @@ -0,0 +1,182 @@ +# Bot Management Patterns + +## E-commerce Protection + +```txt +# High security for checkout +(cf.bot_management.score lt 50 and http.request.uri.path in {"/checkout" "/cart/add"} and not cf.bot_management.verified_bot and not cf.bot_management.corporate_proxy) +Action: Managed Challenge +``` + +## API Protection + +```txt +# Protect API with JS detection + score +(http.request.uri.path matches "^/api/" and (cf.bot_management.score lt 30 or not cf.bot_management.js_detection.passed) and not cf.bot_management.verified_bot) +Action: Block +``` + +## SEO-Friendly Bot Handling + +```txt +# Allow search engine crawlers +(cf.bot_management.score lt 30 and not cf.verified_bot_category in {"Search Engine Crawler"}) +Action: Managed Challenge +``` + +## Block AI Scrapers + +```txt +# Block training crawlers only (allow AI assistants/search) +(cf.verified_bot_category eq "AI Crawler") +Action: Block + +# Block all AI-related bots (training + assistants + search) +(cf.verified_bot_category in {"AI Crawler" "AI Assistant" "AI Search"}) +Action: Block + +# Allow AI Search, block AI Crawler and AI Assistant +(cf.verified_bot_category in {"AI Crawler" "AI Assistant"}) +Action: Block + +# Or use dashboard: Security > Settings > Bot Management > Block AI Bots +``` + +## Rate Limiting by Bot Score + +```txt +# Stricter limits for suspicious traffic +(cf.bot_management.score lt 50) +Rate: 10 requests per 10 seconds + +(cf.bot_management.score ge 50) +Rate: 100 requests per 10 seconds +``` + +## Mobile App Allowlisting + +```txt +# Identify mobile app by JA3/JA4 +(cf.bot_management.ja4 in {"fingerprint1" "fingerprint2"}) +Action: Skip (all remaining rules) +``` + +## Datacenter Detection + +```typescript +import type { IncomingRequestCfProperties } from '@cloudflare/workers-types'; + +// Low score + not corporate proxy = likely datacenter bot +export default { + async fetch(request: Request): Promise { + const cf = request.cf as IncomingRequestCfProperties | undefined; + const botMgmt = cf?.botManagement; + + if (botMgmt?.score && botMgmt.score < 30 && + !botMgmt.corporateProxy && !botMgmt.verifiedBot) { + return new Response('Datacenter traffic blocked', { status: 403 }); + } + + return fetch(request); + } +}; +``` + +## Conditional Delay (Tarpit) + +```typescript +import type { IncomingRequestCfProperties } from '@cloudflare/workers-types'; + +// Add delay proportional to bot suspicion +export default { + async fetch(request: Request): Promise { + const cf = request.cf as IncomingRequestCfProperties | undefined; + const botMgmt = cf?.botManagement; + + if (botMgmt?.score && botMgmt.score < 50 && !botMgmt.verifiedBot) { + // Delay: 0-2 seconds for scores 50-0 + const delayMs = Math.max(0, (50 - botMgmt.score) * 40); + await new Promise(r => setTimeout(r, delayMs)); + } + + return fetch(request); + } +}; +``` + +## Layered Defense + +```txt +1. Bot Management (score-based) +2. JavaScript Detections (for JS-capable clients) +3. Rate Limiting (fallback protection) +4. WAF Managed Rules (OWASP, etc.) +``` + +## Progressive Enhancement + +```txt +Public content: High threshold (score < 10) +Authenticated: Medium threshold (score < 30) +Sensitive: Low threshold (score < 50) + JSD +``` + +## Zero Trust for Bots + +```txt +1. Default deny (all scores < 30) +2. Allowlist verified bots +3. Allowlist mobile apps (JA3/JA4) +4. Allowlist corporate proxies +5. Allowlist static resources +``` + +## Workers: Score + JS Detection + +```typescript +import type { IncomingRequestCfProperties } from '@cloudflare/workers-types'; + +export default { + async fetch(request: Request): Promise { + const cf = request.cf as IncomingRequestCfProperties | undefined; + const botMgmt = cf?.botManagement; + const url = new URL(request.url); + + if (botMgmt?.staticResource) return fetch(request); // Skip static + + // API endpoints: require JS detection + good score + if (url.pathname.startsWith('/api/')) { + const jsDetectionPassed = botMgmt?.jsDetection?.passed ?? false; + const score = botMgmt?.score ?? 100; + + if (!jsDetectionPassed || score < 30) { + return new Response('Unauthorized', { status: 401 }); + } + } + + return fetch(request); + } +}; +``` + +## Rate Limiting by JWT Claim + Bot Score + +```txt +# Enterprise: Combine bot score with JWT validation +Rate limiting > Custom rules +- Field: lookup_json_string(http.request.jwt.claims["{config_id}"][0], "sub") +- Matches: user ID claim +- Additional condition: cf.bot_management.score lt 50 +``` + +## WAF Integration Points + +- **WAF Custom Rules**: Primary enforcement mechanism +- **Rate Limiting Rules**: Bot score as dimension, stricter limits for low scores +- **Transform Rules**: Pass score to origin via custom header +- **Workers**: Programmatic bot logic, custom scoring algorithms +- **Page Rules / Configuration Rules**: Zone-level overrides, path-specific settings + +## See Also + +- [gotchas.md](./gotchas.md) - Common errors, false positives/negatives, limitations diff --git a/plugins/cloudflare/skills/cloudflare/references/browser-rendering/README.md b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/README.md new file mode 100644 index 0000000..eca7220 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/README.md @@ -0,0 +1,78 @@ +# Cloudflare Browser Rendering Skill Reference + +**Description**: Expert knowledge for Cloudflare Browser Rendering - control headless Chrome on Cloudflare's global network for browser automation, screenshots, PDFs, web scraping, testing, and content generation. + +**When to use**: Any task involving Cloudflare Browser Rendering including: taking screenshots, generating PDFs, web scraping, browser automation, testing web applications, extracting structured data, capturing page metrics, or automating browser interactions. + +## Decision Tree + +### REST API vs Workers Bindings + +**Use REST API when:** +- One-off, stateless tasks (screenshot, PDF, content fetch) +- No Workers infrastructure yet +- Simple integrations from external services +- Need quick prototyping without deployment + +**Use Workers Bindings when:** +- Complex browser automation workflows +- Need session reuse for performance +- Multiple page interactions per request +- Custom scripting and logic required +- Building production applications + +### Puppeteer vs Playwright + +| Feature | Puppeteer | Playwright | +|---------|-----------|------------| +| API Style | Chrome DevTools Protocol | High-level abstractions | +| Selectors | CSS, XPath | CSS, text, role, test-id | +| Best for | Advanced control, CDP access | Quick automation, testing | +| Learning curve | Steeper | Gentler | + +**Use Puppeteer:** Need CDP protocol access, Chrome-specific features, migration from existing Puppeteer code +**Use Playwright:** Modern selector APIs, cross-browser patterns, faster development + +## Tier Limits Summary + +| Limit | Free Tier | Paid Tier | +|-------|-----------|-----------| +| Daily browser time | 10 minutes | Unlimited* | +| Concurrent sessions | 3 | 30 | +| Requests per minute | 6 | 180 | + +*Subject to fair-use policy. See [gotchas.md](gotchas.md) for details. + +## Reading Order + +**New to Browser Rendering:** +1. [configuration.md](configuration.md) - Setup and deployment +2. [patterns.md](patterns.md) - Common use cases with examples +3. [api.md](api.md) - API reference +4. [gotchas.md](gotchas.md) - Avoid common pitfalls + +**Specific task:** +- **Setup/deployment** → [configuration.md](configuration.md) +- **API reference/endpoints** → [api.md](api.md) +- **Example code/patterns** → [patterns.md](patterns.md) +- **Debugging/troubleshooting** → [gotchas.md](gotchas.md) + +**REST API users:** +- Start with [api.md](api.md) REST API section +- Check [gotchas.md](gotchas.md) for rate limits + +**Workers users:** +- Start with [configuration.md](configuration.md) +- Review [patterns.md](patterns.md) for session management +- Reference [api.md](api.md) for Workers Bindings + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, wrangler config, compatibility +- **[api.md](api.md)** - REST API endpoints + Workers Bindings (Puppeteer/Playwright) +- **[patterns.md](patterns.md)** - Common patterns, use cases, real examples +- **[gotchas.md](gotchas.md)** - Troubleshooting, best practices, tier limits, common errors + +## See Also + +- [Cloudflare Docs](https://developers.cloudflare.com/browser-rendering/) diff --git a/plugins/cloudflare/skills/cloudflare/references/browser-rendering/api.md b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/api.md new file mode 100644 index 0000000..eea56b0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/api.md @@ -0,0 +1,108 @@ +# Browser Rendering API + +## REST API + +**Base:** `https://api.cloudflare.com/client/v4/accounts/{accountId}/browser-rendering` +**Auth:** `Authorization: Bearer ` (Browser Rendering - Edit permission) + +### Endpoints + +| Endpoint | Description | Key Options | +|----------|-------------|-------------| +| `/content` | Get rendered HTML | `url`, `waitUntil` | +| `/screenshot` | Capture image | `screenshotOptions: {type, fullPage, clip}` | +| `/pdf` | Generate PDF | `pdfOptions: {format, landscape, margin}` | +| `/snapshot` | HTML + inlined resources | `url` | +| `/scrape` | Extract by selectors | `selectors: ["h1", ".price"]` | +| `/json` | AI-structured extraction | `schema: {name: "string", price: "number"}` | +| `/links` | Get all links | `url` | +| `/markdown` | Convert to markdown | `url` | + +```bash +curl -X POST '.../browser-rendering/screenshot' \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"url":"https://example.com","screenshotOptions":{"fullPage":true}}' +``` + +## Workers Binding + +```jsonc +// wrangler.jsonc +{ "browser": { "binding": "MYBROWSER" } } +``` + +## Puppeteer + +```typescript +import puppeteer from "@cloudflare/puppeteer"; + +const browser = await puppeteer.launch(env.MYBROWSER, { keep_alive: 600000 }); +const page = await browser.newPage(); +await page.goto('https://example.com', { waitUntil: 'networkidle0' }); + +// Content +const html = await page.content(); +const title = await page.title(); + +// Screenshot/PDF +await page.screenshot({ fullPage: true, type: 'png' }); +await page.pdf({ format: 'A4', printBackground: true }); + +// Interaction +await page.click('#button'); +await page.type('#input', 'text'); +await page.evaluate(() => document.querySelector('h1')?.textContent); + +// Session management +const sessions = await puppeteer.sessions(env.MYBROWSER); +const limits = await puppeteer.limits(env.MYBROWSER); + +await browser.close(); +``` + +## Playwright + +```typescript +import { launch, connect } from "@cloudflare/playwright"; + +const browser = await launch(env.MYBROWSER, { keep_alive: 600000 }); +const page = await browser.newPage(); + +await page.goto('https://example.com', { waitUntil: 'networkidle' }); + +// Modern selectors +await page.locator('.button').click(); +await page.getByText('Submit').click(); +await page.getByTestId('search').fill('query'); + +// Context for isolation +const context = await browser.newContext({ + viewport: { width: 1920, height: 1080 }, + userAgent: 'custom' +}); + +await browser.close(); +``` + +## Session Management + +```typescript +// List sessions +await puppeteer.sessions(env.MYBROWSER); + +// Connect to existing +await puppeteer.connect(env.MYBROWSER, sessionId); + +// Check limits +await puppeteer.limits(env.MYBROWSER); +// { remaining: ms, total: ms, concurrent: n } +``` + +## Key Options + +| Option | Values | +|--------|--------| +| `waitUntil` | `load`, `domcontentloaded`, `networkidle0`, `networkidle2` | +| `keep_alive` | Max 600000ms (10 min) | +| `screenshot.type` | `png`, `jpeg` | +| `pdf.format` | `A4`, `Letter`, `Legal` | diff --git a/plugins/cloudflare/skills/cloudflare/references/browser-rendering/configuration.md b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/configuration.md new file mode 100644 index 0000000..84bad26 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/configuration.md @@ -0,0 +1,78 @@ +# Configuration & Setup + +## Installation + +```bash +npm install @cloudflare/puppeteer # or @cloudflare/playwright +``` + +**Use Cloudflare packages** - standard `puppeteer`/`playwright` won't work in Workers. + +## wrangler.json + +```json +{ + "name": "browser-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", + "compatibility_flags": ["nodejs_compat"], + "browser": { + "binding": "MYBROWSER" + } +} +``` + +**Required:** `nodejs_compat` flag and `browser.binding`. + +## TypeScript + +```typescript +interface Env { + MYBROWSER: Fetcher; +} + +export default { + async fetch(request: Request, env: Env): Promise { + // ... + } +} satisfies ExportedHandler; +``` + +## Development + +```bash +wrangler dev --remote # --remote required for browser binding +``` + +**Local mode does NOT support Browser Rendering** - must use `--remote`. + +## REST API + +No wrangler config needed. Get API token with "Browser Rendering - Edit" permission. + +```bash +curl -X POST \ + 'https://api.cloudflare.com/client/v4/accounts/{accountId}/browser-rendering/screenshot' \ + -H 'Authorization: Bearer TOKEN' \ + -d '{"url": "https://example.com"}' --output screenshot.png +``` + +## Requirements + +| Requirement | Value | +|-------------|-------| +| Node.js compatibility | `nodejs_compat` flag | +| Compatibility date | 2023-03-01+ | +| Module format | ES modules only | +| Browser | Chromium 119+ (no Firefox/Safari) | + +**Not supported:** WebGL, WebRTC, extensions, `file://` protocol, Service Worker syntax. + +## Troubleshooting + +| Error | Solution | +|-------|----------| +| `MYBROWSER is undefined` | Use `wrangler dev --remote` | +| `nodejs_compat not enabled` | Add to `compatibility_flags` | +| `Module not found` | `npm install @cloudflare/puppeteer` | +| `Browser Rendering not available` | Enable in dashboard | diff --git a/plugins/cloudflare/skills/cloudflare/references/browser-rendering/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/gotchas.md new file mode 100644 index 0000000..7e34f2b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/gotchas.md @@ -0,0 +1,88 @@ +# Browser Rendering Gotchas + +## Tier Limits + +| Limit | Free | Paid | +|-------|------|------| +| Daily browser time | 10 min | Unlimited* | +| Concurrent sessions | 3 | 30 | +| Requests/minute | 6 | 180 | +| Session keep-alive | 10 min max | 10 min max | + +*Subject to fair-use policy. + +**Check quota:** +```typescript +const limits = await puppeteer.limits(env.MYBROWSER); +// { remaining: 540000, total: 600000, concurrent: 2 } +``` + +## Always Close Browsers + +```typescript +const browser = await puppeteer.launch(env.MYBROWSER); +try { + const page = await browser.newPage(); + await page.goto("https://example.com"); + return new Response(await page.content()); +} finally { + await browser.close(); // ALWAYS in finally +} +``` + +**Workers vs REST:** REST auto-closes after timeout. Workers must call `close()` or session stays open until `keep_alive` expires. + +## Optimize Concurrency + +```typescript +// ❌ 3 sessions (hits free tier limit) +const browser1 = await puppeteer.launch(env.MYBROWSER); +const browser2 = await puppeteer.launch(env.MYBROWSER); + +// ✅ 1 session, multiple pages +const browser = await puppeteer.launch(env.MYBROWSER); +const page1 = await browser.newPage(); +const page2 = await browser.newPage(); +``` + +## Common Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| Session limit exceeded | Too many concurrent | Close unused browsers, use pages not browsers | +| Page navigation timeout | Slow page or `networkidle` on busy page | Increase timeout, use `waitUntil: "load"` | +| Session not found | Expired session | Catch error, launch new session | +| Evaluation failed | DOM element missing | Use `?.` optional chaining | +| Protocol error: Target closed | Page closed during operation | Await all ops before closing | + +## page.evaluate() Gotchas + +```typescript +// ❌ Outer scope not available +const selector = "h1"; +await page.evaluate(() => document.querySelector(selector)); + +// ✅ Pass as argument +await page.evaluate((sel) => document.querySelector(sel)?.textContent, selector); +``` + +## Performance + +**waitUntil options (fastest to slowest):** +1. `domcontentloaded` - DOM ready +2. `load` - load event (default) +3. `networkidle0` - no network for 500ms + +**Block unnecessary resources:** +```typescript +await page.setRequestInterception(true); +page.on("request", (req) => { + if (["image", "stylesheet", "font"].includes(req.resourceType())) { + req.abort(); + } else { + req.continue(); + } +}); +``` + +**Session reuse:** Cold start ~1-2s, warm connect ~100-200ms. Store sessionId in KV for reuse. diff --git a/plugins/cloudflare/skills/cloudflare/references/browser-rendering/patterns.md b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/patterns.md new file mode 100644 index 0000000..a652c2b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/browser-rendering/patterns.md @@ -0,0 +1,91 @@ +# Browser Rendering Patterns + +## Basic Worker + +```typescript +import puppeteer from "@cloudflare/puppeteer"; + +export default { + async fetch(request, env) { + const browser = await puppeteer.launch(env.MYBROWSER); + try { + const page = await browser.newPage(); + await page.goto("https://example.com"); + return new Response(await page.content()); + } finally { + await browser.close(); // ALWAYS in finally + } + } +}; +``` + +## Session Reuse + +Keep sessions alive for performance: +```typescript +let sessionId = await env.SESSION_KV.get("browser-session"); +if (sessionId) { + browser = await puppeteer.connect(env.MYBROWSER, sessionId); +} else { + browser = await puppeteer.launch(env.MYBROWSER, { keep_alive: 600000 }); + await env.SESSION_KV.put("browser-session", browser.sessionId(), { expirationTtl: 600 }); +} +// Don't close browser to keep session alive +``` + +## Common Operations + +| Task | Code | +|------|------| +| Screenshot | `await page.screenshot({ type: "png", fullPage: true })` | +| PDF | `await page.pdf({ format: "A4", printBackground: true })` | +| Extract data | `await page.evaluate(() => document.querySelector('h1').textContent)` | +| Fill form | `await page.type('#input', 'value'); await page.click('button')` | +| Wait nav | `await Promise.all([page.waitForNavigation(), page.click('a')])` | + +## Parallel Scraping + +```typescript +const pages = await Promise.all(urls.map(() => browser.newPage())); +await Promise.all(pages.map((p, i) => p.goto(urls[i]))); +const titles = await Promise.all(pages.map(p => p.title())); +``` + +## Playwright Selectors + +```typescript +import { launch } from "@cloudflare/playwright"; +const browser = await launch(env.MYBROWSER); +await page.getByRole("button", { name: "Sign in" }).click(); +await page.getByLabel("Email").fill("user@example.com"); +await page.getByTestId("submit-button").click(); +``` + +## Incognito Contexts + +Isolated sessions without multiple browsers: +```typescript +const ctx1 = await browser.createIncognitoBrowserContext(); +const ctx2 = await browser.createIncognitoBrowserContext(); +// Each has isolated cookies/storage +``` + +## Quota Check + +```typescript +const limits = await puppeteer.limits(env.MYBROWSER); +if (limits.remaining < 60000) return new Response("Quota low", { status: 429 }); +``` + +## Error Handling + +```typescript +try { + await page.goto(url, { timeout: 30000, waitUntil: "networkidle0" }); +} catch (e) { + if (e.message.includes("timeout")) return new Response("Timeout", { status: 504 }); + if (e.message.includes("Session limit")) return new Response("Too many sessions", { status: 429 }); +} finally { + if (browser) await browser.close(); +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/c3/README.md b/plugins/cloudflare/skills/cloudflare/references/c3/README.md new file mode 100644 index 0000000..0516fc6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/c3/README.md @@ -0,0 +1,111 @@ +# C3 (create-cloudflare) + +Official CLI for scaffolding Cloudflare Workers and Pages projects with templates, TypeScript, and instant deployment. + +## Quick Start + +```bash +# Interactive (recommended for first-time) +npm create cloudflare@latest my-app + +# Worker (API/WebSocket/Cron) +npm create cloudflare@latest my-api -- --type=hello-world --ts + +# Pages (static/SSG/full-stack) +npm create cloudflare@latest my-site -- --type=web-app --framework=astro --platform=pages +``` + +## Platform Decision Tree + +``` +What are you building? + +├─ API / WebSocket / Cron / Email handler +│ └─ Workers (default) - no --platform flag needed +│ npm create cloudflare@latest my-api -- --type=hello-world + +├─ Static site / SSG / Documentation +│ └─ Pages - requires --platform=pages +│ npm create cloudflare@latest my-site -- --type=web-app --framework=astro --platform=pages + +├─ Full-stack app (Next.js/Remix/SvelteKit) +│ ├─ Need Durable Objects, Queues, or Workers-only features? +│ │ └─ Workers (default) +│ └─ Otherwise use Pages for git integration and branch previews +│ └─ Add --platform=pages + +└─ Convert existing project + └─ npm create cloudflare@latest . -- --type=pre-existing --existing-script=./src/worker.ts +``` + +**Critical:** Pages projects require `--platform=pages` flag. Without it, C3 defaults to Workers. + +## Interactive Flow + +When run without flags, C3 prompts in this order: + +1. **Project name** - Directory to create (defaults to current dir with `.`) +2. **Application type** - `hello-world`, `web-app`, `demo`, `pre-existing`, `remote-template` +3. **Platform** - `workers` (default) or `pages` (for web apps only) +4. **Framework** - If web-app: `next`, `remix`, `astro`, `react-router`, `solid`, `svelte`, etc. +5. **TypeScript** - `yes` (recommended) or `no` +6. **Git** - Initialize repository? `yes` or `no` +7. **Deploy** - Deploy now? `yes` or `no` (requires `wrangler login`) + +## Installation Methods + +```bash +# NPM +npm create cloudflare@latest + +# Yarn +yarn create cloudflare + +# PNPM +pnpm create cloudflare@latest +``` + +## In This Reference + +| File | Purpose | Use When | +|------|---------|----------| +| **api.md** | Complete CLI flag reference | Scripting, CI/CD, advanced usage | +| **configuration.md** | Generated files, bindings, types | Understanding output, customization | +| **patterns.md** | Workflows, CI/CD, monorepos | Real-world integration | +| **gotchas.md** | Troubleshooting failures | Deployment blocked, errors | + +## Reading Order + +| Task | Read | +|------|------| +| Create first project | README only | +| Set up CI/CD | README → api → patterns | +| Debug failed deploy | gotchas | +| Understand generated files | configuration | +| Full CLI reference | api | +| Create custom template | patterns → configuration | +| Convert existing project | README → patterns | + +## Post-Creation + +```bash +cd my-app + +# Local dev with hot reload +npm run dev + +# Generate TypeScript types for bindings +npm run cf-typegen + +# Deploy to Cloudflare +npm run deploy +``` + +## See Also + +- **workers/README.md** - Workers runtime, bindings, APIs +- **workers-ai/README.md** - AI/ML models +- **pages/README.md** - Pages-specific features +- **wrangler/README.md** - Wrangler CLI beyond initial setup +- **d1/README.md** - SQLite database +- **r2/README.md** - Object storage diff --git a/plugins/cloudflare/skills/cloudflare/references/c3/api.md b/plugins/cloudflare/skills/cloudflare/references/c3/api.md new file mode 100644 index 0000000..29c2b0c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/c3/api.md @@ -0,0 +1,71 @@ +# C3 CLI Reference + +## Invocation + +```bash +npm create cloudflare@latest [name] [-- flags] # NPM requires -- +yarn create cloudflare [name] [flags] +pnpm create cloudflare@latest [name] [-- flags] +``` + +## Core Flags + +| Flag | Values | Description | +|------|--------|-------------| +| `--type` | `hello-world`, `web-app`, `demo`, `pre-existing`, `remote-template` | Application type | +| `--platform` | `workers` (default), `pages` | Target platform | +| `--framework` | `next`, `remix`, `astro`, `react-router`, `solid`, `svelte`, `qwik`, `vue`, `angular`, `hono` | Web framework (requires `--type=web-app`) | +| `--lang` | `ts`, `js`, `python` | Language (for `--type=hello-world`) | +| `--ts` / `--no-ts` | - | TypeScript for web apps | + +## Deployment Flags + +| Flag | Description | +|------|-------------| +| `--deploy` / `--no-deploy` | Deploy immediately (prompts interactive, skips in CI) | +| `--git` / `--no-git` | Initialize git (default: yes) | +| `--open` | Open browser after deploy | + +## Advanced Flags + +| Flag | Description | +|------|-------------| +| `--template=user/repo` | GitHub template or local path | +| `--existing-script=./src/worker.ts` | Existing script (requires `--type=pre-existing`) | +| `--category=ai\|database\|realtime` | Demo filter (requires `--type=demo`) | +| `--experimental` | Enable experimental features | +| `--wrangler-defaults` | Skip wrangler prompts | + +## Environment Variables + +```bash +CLOUDFLARE_API_TOKEN=xxx # For deployment +CLOUDFLARE_ACCOUNT_ID=xxx # Account ID +CF_TELEMETRY_DISABLED=1 # Disable telemetry +``` + +## Exit Codes + +`0` success, `1` user abort, `2` error + +## Examples + +```bash +# TypeScript Worker +npm create cloudflare@latest my-api -- --type=hello-world --lang=ts --no-deploy + +# Next.js on Pages +npm create cloudflare@latest my-app -- --type=web-app --framework=next --platform=pages --ts + +# Astro blog +npm create cloudflare@latest my-blog -- --type=web-app --framework=astro --ts --deploy + +# CI: non-interactive +npm create cloudflare@latest my-app -- --type=web-app --framework=next --ts --no-git --no-deploy + +# GitHub template +npm create cloudflare@latest -- --template=cloudflare/templates/worker-openapi + +# Convert existing project +npm create cloudflare@latest . -- --type=pre-existing --existing-script=./build/worker.js +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/c3/configuration.md b/plugins/cloudflare/skills/cloudflare/references/c3/configuration.md new file mode 100644 index 0000000..37f9f82 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/c3/configuration.md @@ -0,0 +1,81 @@ +# C3 Generated Configuration + +## Output Structure + +``` +my-app/ +├── src/index.ts # Worker entry point +├── wrangler.jsonc # Cloudflare config +├── package.json # Scripts +├── tsconfig.json +└── .gitignore +``` + +## wrangler.jsonc + +```jsonc +{ + "$schema": "https://raw.githubusercontent.com/cloudflare/workers-sdk/main/packages/wrangler/config-schema.json", + "name": "my-app", + "main": "src/index.ts", + "compatibility_date": "2026-01-27" +} +``` + +## Binding Placeholders + +C3 generates **placeholder IDs** that must be replaced before deploy: + +```jsonc +{ + "kv_namespaces": [{ "binding": "MY_KV", "id": "placeholder_kv_id" }], + "d1_databases": [{ "binding": "DB", "database_id": "00000000-..." }] +} +``` + +**Replace with real IDs:** +```bash +npx wrangler kv namespace create MY_KV # Returns real ID +npx wrangler d1 create my-database # Returns real database_id +``` + +**Deployment error if not replaced:** +``` +Error: Invalid KV namespace ID "placeholder_kv_id" +``` + +## Scripts + +```json +{ + "scripts": { + "dev": "wrangler dev", + "deploy": "wrangler deploy", + "cf-typegen": "wrangler types" + } +} +``` + +## Type Generation + +Run after adding bindings: +```bash +npm run cf-typegen +``` + +Generates `.wrangler/types/runtime.d.ts`: +```typescript +interface Env { + MY_KV: KVNamespace; + DB: D1Database; +} +``` + +## Post-Creation Checklist + +1. Review `wrangler.jsonc` - check name, compatibility_date +2. Replace placeholder binding IDs with real resource IDs +3. Run `npm run cf-typegen` +4. Test: `npm run dev` +5. Deploy: `npm run deploy` +6. Add secrets: `npx wrangler secret put SECRET_NAME` diff --git a/plugins/cloudflare/skills/cloudflare/references/c3/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/c3/gotchas.md new file mode 100644 index 0000000..ecd664d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/c3/gotchas.md @@ -0,0 +1,92 @@ +# C3 Troubleshooting + +## Deployment Issues + +### Placeholder IDs + +**Error:** "Invalid namespace ID" +**Fix:** Replace placeholders in wrangler.jsonc with real IDs: +```bash +npx wrangler kv namespace create MY_KV # Get real ID +``` + +### Authentication + +**Error:** "Not authenticated" +**Fix:** `npx wrangler login` or set `CLOUDFLARE_API_TOKEN` + +### Name Conflict + +**Error:** "Worker already exists" +**Fix:** Change `name` in wrangler.jsonc + +## Platform Selection + +| Need | Platform | +|------|----------| +| Git integration, branch previews | `--platform=pages` | +| Durable Objects, D1, Queues | Workers (default) | + +Wrong platform? Recreate with correct `--platform` flag. + +## TypeScript Issues + +**"Cannot find name 'KVNamespace'"** +```bash +npm run cf-typegen # Regenerate types +# Restart TS server in editor +``` + +**Missing types after config change:** Re-run `npm run cf-typegen` + +## Package Manager + +**Multiple lockfiles causing issues:** +```bash +rm pnpm-lock.yaml # If using npm +rm package-lock.json # If using pnpm +``` + +## CI/CD + +**CI hangs on prompts:** +```bash +npm create cloudflare@latest my-app -- \ + --type=hello-world --lang=ts --no-git --no-deploy +``` + +**Auth in CI:** +```yaml +env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} +``` + +## Framework-Specific + +| Framework | Issue | Fix | +|-----------|-------|-----| +| Next.js | create-next-app failed | `npm cache clean --force`, retry | +| Astro | Adapter missing | Install `@astrojs/cloudflare` | +| Remix | Module errors | Update `@remix-run/cloudflare*` | + +## Compatibility Date + +**"Feature X requires compatibility_date >= ..."** +**Fix:** Update `compatibility_date` in wrangler.jsonc to today's date + +## Node.js Version + +**"Node.js version not supported"** +**Fix:** Install Node.js 18+ (`nvm install 20`) + +## Quick Reference + +| Error | Cause | Fix | +|-------|-------|-----| +| Invalid namespace ID | Placeholder binding | Create resource, update config | +| Not authenticated | No login | `npx wrangler login` | +| Cannot find KVNamespace | Missing types | `npm run cf-typegen` | +| Worker already exists | Name conflict | Change `name` | +| CI hangs | Missing flags | Add --type, --lang, --no-deploy | +| Template not found | Bad name | Check cloudflare/templates | diff --git a/plugins/cloudflare/skills/cloudflare/references/c3/patterns.md b/plugins/cloudflare/skills/cloudflare/references/c3/patterns.md new file mode 100644 index 0000000..76379e3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/c3/patterns.md @@ -0,0 +1,82 @@ +# C3 Usage Patterns + +## Quick Workflows + +```bash +# TypeScript API Worker +npm create cloudflare@latest my-api -- --type=hello-world --lang=ts --deploy + +# Next.js on Pages +npm create cloudflare@latest my-app -- --type=web-app --framework=next --platform=pages --ts --deploy + +# Astro static site +npm create cloudflare@latest my-blog -- --type=web-app --framework=astro --platform=pages --ts +``` + +## CI/CD (GitHub Actions) + +```yaml +- name: Deploy + run: npm run deploy + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} +``` + +**Non-interactive requires:** +```bash +--type= # Required +--no-git # Recommended (CI already in git) +--no-deploy # Deploy separately with secrets +--framework= # For web-app +--ts / --no-ts # Required +``` + +## Monorepo + +C3 detects workspace config (`package.json` workspaces or `pnpm-workspace.yaml`). + +```bash +cd packages/ +npm create cloudflare@latest my-worker -- --type=hello-world --lang=ts --no-deploy +``` + +## Custom Templates + +```bash +# GitHub repo +npm create cloudflare@latest -- --template=username/repo +npm create cloudflare@latest -- --template=cloudflare/templates/worker-openapi + +# Local path +npm create cloudflare@latest my-app -- --template=../my-template +``` + +**Template requires `c3.config.json`:** +```json +{ + "name": "my-template", + "category": "hello-world", + "copies": [{ "path": "src/" }, { "path": "wrangler.jsonc" }], + "transforms": [{ "path": "package.json", "jsonc": { "name": "{{projectName}}" }}] +} +``` + +## Existing Projects + +```bash +# Add Cloudflare to existing Worker +npm create cloudflare@latest . -- --type=pre-existing --existing-script=./dist/index.js + +# Add to existing framework app +npm create cloudflare@latest . -- --type=web-app --framework=next --platform=pages --ts +``` + +## Post-Creation Checklist + +1. Review `wrangler.jsonc` - set `compatibility_date`, verify `name` +2. Create bindings: `wrangler kv namespace create`, `wrangler d1 create`, `wrangler r2 bucket create` +3. Generate types: `npm run cf-typegen` +4. Test: `npm run dev` +5. Deploy: `npm run deploy` +6. Set secrets: `wrangler secret put SECRET_NAME` diff --git a/plugins/cloudflare/skills/cloudflare/references/cache-reserve/README.md b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/README.md new file mode 100644 index 0000000..395347a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/README.md @@ -0,0 +1,147 @@ +# Cloudflare Cache Reserve + +**Persistent cache storage built on R2 for long-term content retention** + +## Smart Shield Integration + +Cache Reserve is part of **Smart Shield**, Cloudflare's comprehensive security and performance suite: + +- **Smart Shield Advanced tier**: Includes 2TB Cache Reserve storage +- **Standalone purchase**: Available separately if not using Smart Shield +- **Migration**: Existing standalone customers can migrate to Smart Shield bundles + +**Decision**: Already on Smart Shield Advanced? Cache Reserve is included. Otherwise evaluate standalone purchase vs Smart Shield upgrade. + +## Overview + +Cache Reserve is Cloudflare's persistent, large-scale cache storage layer built on R2. It acts as the ultimate upper-tier cache, storing cacheable content for extended periods (30+ days) to maximize cache hits, reduce origin egress fees, and shield origins from repeated requests for long-tail content. + +## Core Concepts + +### What is Cache Reserve? + +- **Persistent storage layer**: Built on R2, sits above tiered cache hierarchy +- **Long-term retention**: 30-day default retention, extended on each access +- **Automatic operation**: Works seamlessly with existing CDN, no code changes required +- **Origin shielding**: Dramatically reduces origin egress by serving cached content longer +- **Usage-based pricing**: Pay only for storage + read/write operations + +### Cache Hierarchy + +``` +Visitor Request + ↓ +Lower-Tier Cache (closest to visitor) + ↓ (on miss) +Upper-Tier Cache (closest to origin) + ↓ (on miss) +Cache Reserve (R2 persistent storage) + ↓ (on miss) +Origin Server +``` + +### How It Works + +1. **On cache miss**: Content fetched from origin �� written to Cache Reserve + edge caches simultaneously +2. **On edge eviction**: Content may be evicted from edge cache but remains in Cache Reserve +3. **On subsequent request**: If edge cache misses but Cache Reserve hits → content restored to edge caches +4. **Retention**: Assets remain in Cache Reserve for 30 days since last access (configurable via TTL) + +## When to Use Cache Reserve + +``` +Need persistent caching? +├─ High origin egress costs → Cache Reserve ✓ +├─ Long-tail content (archives, media libraries) → Cache Reserve ✓ +├─ Already using Smart Shield Advanced → Included! ✓ +├─ Video streaming with seeking (range requests) → ✗ Not supported +├─ Dynamic/personalized content → ✗ Use edge cache only +├─ Need per-request cache control from Workers → ✗ Use R2 directly +└─ Frequently updated content (< 10hr lifetime) → ✗ Not eligible +``` + +## Asset Eligibility + +Cache Reserve only stores assets meeting **ALL** criteria: + +- Cacheable per Cloudflare's standard rules +- Minimum 10-hour TTL (36000 seconds) +- `Content-Length` header present +- Original files only (not transformed images) + +### Eligibility Checklist + +Use this checklist to verify if an asset is eligible: + +- [ ] Zone has Cache Reserve enabled +- [ ] Zone has Tiered Cache enabled (required) +- [ ] Asset TTL ≥ 10 hours (36,000 seconds) +- [ ] `Content-Length` header present on origin response +- [ ] No `Set-Cookie` header (or uses private directive) +- [ ] `Vary` header is NOT `*` (can be `Accept-Encoding`) +- [ ] Not an image transformation variant (original images OK) +- [ ] Not a range request (no HTTP 206 support) +- [ ] Not O2O (Orange-to-Orange) proxied request + +**All boxes must be checked for Cache Reserve eligibility.** + +### Not Eligible + +- Assets with TTL < 10 hours +- Responses without `Content-Length` header +- Image transformation variants (original images are eligible) +- Responses with `Set-Cookie` headers +- Responses with `Vary: *` header +- Assets from R2 public buckets on same zone +- O2O (Orange-to-Orange) setup requests +- **Range requests** (video seeking, partial content downloads) + +## Quick Start + +```bash +# Enable via Dashboard +https://dash.cloudflare.com/caching/cache-reserve +# Click "Enable Storage Sync" or "Purchase" button +``` + +**Prerequisites:** +- Paid Cache Reserve plan or Smart Shield Advanced required +- Tiered Cache required for optimal performance + +## Essential Commands + +```bash +# Check Cache Reserve status +curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/cache/cache_reserve" \ + -H "Authorization: Bearer $API_TOKEN" + +# Enable Cache Reserve +curl -X PATCH "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/cache/cache_reserve" \ + -H "Authorization: Bearer $API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"value": "on"}' + +# Check asset cache status +curl -I https://example.com/asset.jpg | grep -i cache +``` + +## In This Reference + +| Task | Files | +|------|-------| +| Evaluate if Cache Reserve fits your use case | README.md (this file) | +| Enable Cache Reserve for your zone | README.md + [configuration.md](./configuration.md) | +| Use with Workers (understand limitations) | [api.md](./api.md) | +| Setup via SDKs or IaC (TypeScript, Python, Terraform) | [configuration.md](./configuration.md) | +| Optimize costs and debug issues | [patterns.md](./patterns.md) + [gotchas.md](./gotchas.md) | +| Understand eligibility and troubleshoot | [gotchas.md](./gotchas.md) → [patterns.md](./patterns.md) | + +**Files:** +- [configuration.md](./configuration.md) - Setup, API, SDKs, and Cache Rules +- [api.md](./api.md) - Purging, monitoring, Workers integration +- [patterns.md](./patterns.md) - Best practices, cost optimization, debugging +- [gotchas.md](./gotchas.md) - Common issues, limitations, troubleshooting + +## See Also +- [r2](../r2/) - Cache Reserve built on R2 storage +- [workers](../workers/) - Workers integration with Cache API diff --git a/plugins/cloudflare/skills/cloudflare/references/cache-reserve/api.md b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/api.md new file mode 100644 index 0000000..18c49d8 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/api.md @@ -0,0 +1,194 @@ +# Cache Reserve API + +## Workers Integration + +``` +┌────────────────────────────────────────────────────────────────┐ +│ CRITICAL: Workers Cache API ≠ Cache Reserve │ +│ │ +│ • Workers caches.default / cache.put() → edge cache ONLY │ +│ • Cache Reserve → zone-level setting, automatic, no per-req │ +│ • You CANNOT selectively write to Cache Reserve from Workers │ +│ • Cache Reserve works with standard fetch(), not cache.put() │ +└────────────────────────────────────────────────────────────────┘ +``` + +Cache Reserve is a **zone-level configuration**, not a per-request API. It works automatically when enabled for the zone: + +### Standard Fetch (Recommended) + +```typescript +// Cache Reserve works automatically via standard fetch +export default { + async fetch(request: Request, env: Env): Promise { + // Standard fetch uses Cache Reserve automatically + return await fetch(request); + } +}; +``` + +### Cache API Limitations + +**IMPORTANT**: `cache.put()` is **NOT compatible** with Cache Reserve or Tiered Cache. + +```typescript +// ❌ WRONG: cache.put() bypasses Cache Reserve +const cache = caches.default; +let response = await cache.match(request); +if (!response) { + response = await fetch(request); + await cache.put(request, response.clone()); // Bypasses Cache Reserve! +} + +// ✅ CORRECT: Use standard fetch for Cache Reserve compatibility +return await fetch(request); + +// ✅ CORRECT: Use Cache API only for custom cache namespaces +const customCache = await caches.open('my-custom-cache'); +let response = await customCache.match(request); +if (!response) { + response = await fetch(request); + await customCache.put(request, response.clone()); // Custom cache OK +} +``` + +## Purging and Cache Management + +### Purge by URL (Instant) + +```typescript +// Purge specific URL from Cache Reserve immediately +const purgeCacheReserveByURL = async ( + zoneId: string, + apiToken: string, + urls: string[] +) => { + const response = await fetch( + `https://api.cloudflare.com/client/v4/zones/${zoneId}/purge_cache`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ files: urls }) + } + ); + return await response.json(); +}; + +// Example usage +await purgeCacheReserveByURL('zone123', 'token456', [ + 'https://example.com/image.jpg', + 'https://example.com/video.mp4' +]); +``` + +### Purge by Tag/Host/Prefix (Revalidation) + +```typescript +// Purge by cache tag - forces revalidation, not immediate removal +await fetch( + `https://api.cloudflare.com/client/v4/zones/${zoneId}/purge_cache`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ tags: ['tag1', 'tag2'] }) + } +); +``` + +**Purge behavior:** +- **By URL**: Immediate removal from Cache Reserve + edge cache +- **By tag/host/prefix**: Revalidation only, assets remain in storage (costs continue) + +### Clear All Cache Reserve Data + +```typescript +// Requires Cache Reserve OFF first +await fetch( + `https://api.cloudflare.com/client/v4/zones/${zoneId}/cache/cache_reserve_clear`, + { method: 'POST', headers: { 'Authorization': `Bearer ${apiToken}` } } +); + +// Check status: GET same endpoint returns { state: "In-progress" | "Completed" } +``` + +**Process**: Disable Cache Reserve → Call clear endpoint → Wait up to 24hr → Re-enable + +## Monitoring and Analytics + +### Dashboard Analytics + +Navigate to **Caching > Cache Reserve** to view: + +- **Egress Savings**: Total bytes served from Cache Reserve vs origin egress cost saved +- **Requests Served**: Cache Reserve hits vs misses breakdown +- **Storage Used**: Current GB stored in Cache Reserve (billed monthly) +- **Operations**: Class A (writes) and Class B (reads) operation counts +- **Cost Tracking**: Estimated monthly costs based on current usage + +### Logpush Integration + +```typescript +// Logpush field: CacheReserveUsed (boolean) - filter for Cache Reserve hits +// Query Cache Reserve hits in analytics +const logpushQuery = ` + SELECT + ClientRequestHost, + COUNT(*) as requests, + SUM(EdgeResponseBytes) as bytes_served, + COUNT(CASE WHEN CacheReserveUsed = true THEN 1 END) as cache_reserve_hits, + COUNT(CASE WHEN CacheReserveUsed = false THEN 1 END) as cache_reserve_misses + FROM http_requests + WHERE Timestamp >= NOW() - INTERVAL '24 hours' + GROUP BY ClientRequestHost + ORDER BY requests DESC +`; + +// Filter only Cache Reserve hits +const crHitsQuery = ` + SELECT ClientRequestHost, COUNT(*) as requests, SUM(EdgeResponseBytes) as bytes + FROM http_requests + WHERE CacheReserveUsed = true AND Timestamp >= NOW() - INTERVAL '7 days' + GROUP BY ClientRequestHost + ORDER BY bytes DESC +`; +``` + +### GraphQL Analytics + +```graphql +query CacheReserveAnalytics($zoneTag: string, $since: string, $until: string) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + httpRequests1dGroups( + filter: { datetime_geq: $since, datetime_leq: $until } + limit: 1000 + ) { + dimensions { date } + sum { + cachedBytes + cachedRequests + bytes + requests + } + } + } + } +} +``` + +## Pricing + +```typescript +// Storage: $0.015/GB-month | Class A (writes): $4.50/M | Class B (reads): $0.36/M +// Cache miss: 1A + 1B | Cache hit: 1B | Assets >1GB: proportionally more ops +``` + +## See Also + +- [README](./README.md) - Overview and core concepts +- [Configuration](./configuration.md) - Setup and Cache Rules +- [Patterns](./patterns.md) - Best practices and optimization +- [Gotchas](./gotchas.md) - Common issues and troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/cache-reserve/configuration.md b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/configuration.md new file mode 100644 index 0000000..84a6616 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/configuration.md @@ -0,0 +1,169 @@ +# Cache Reserve Configuration + +## Dashboard Setup + +**Minimum steps to enable:** + +```bash +# Navigate to dashboard +https://dash.cloudflare.com/caching/cache-reserve + +# Click "Enable Storage Sync" or "Purchase" button +``` + +**Prerequisites:** +- Paid Cache Reserve plan or Smart Shield Advanced required +- Tiered Cache **required** for Cache Reserve to function optimally + +## API Configuration + +### REST API + +```bash +# Enable +curl -X PATCH "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/cache/cache_reserve" \ + -H "Authorization: Bearer $API_TOKEN" -H "Content-Type: application/json" \ + -d '{"value": "on"}' + +# Check status +curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/cache/cache_reserve" \ + -H "Authorization: Bearer $API_TOKEN" +``` + +### TypeScript SDK + +```bash +npm install cloudflare +``` + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ + apiToken: process.env.CLOUDFLARE_API_TOKEN, +}); + +// Enable Cache Reserve +await client.cache.cacheReserve.edit({ + zone_id: 'abc123', + value: 'on', +}); + +// Get Cache Reserve status +const status = await client.cache.cacheReserve.get({ + zone_id: 'abc123', +}); +console.log(status.value); // 'on' or 'off' +``` + +### Python SDK + +```bash +pip install cloudflare +``` + +```python +from cloudflare import Cloudflare + +client = Cloudflare(api_token=os.environ.get("CLOUDFLARE_API_TOKEN")) + +# Enable Cache Reserve +client.cache.cache_reserve.edit( + zone_id="abc123", + value="on" +) + +# Get Cache Reserve status +status = client.cache.cache_reserve.get(zone_id="abc123") +print(status.value) # 'on' or 'off' +``` + +### Terraform + +```hcl +terraform { + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 4.0" + } + } +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +resource "cloudflare_zone_cache_reserve" "example" { + zone_id = var.zone_id + enabled = true +} + +# Tiered Cache is required for Cache Reserve +resource "cloudflare_tiered_cache" "example" { + zone_id = var.zone_id + cache_type = "smart" +} +``` + +### Pulumi + +```typescript +import * as cloudflare from "@pulumi/cloudflare"; + +// Enable Cache Reserve +const cacheReserve = new cloudflare.ZoneCacheReserve("example", { + zoneId: zoneId, + enabled: true, +}); + +// Enable Tiered Cache (required) +const tieredCache = new cloudflare.TieredCache("example", { + zoneId: zoneId, + cacheType: "smart", +}); +``` + +### Required API Token Permissions + +- `Zone Settings Read` +- `Zone Settings Write` +- `Zone Read` +- `Zone Write` + +## Cache Rules Integration + +Control Cache Reserve eligibility via Cache Rules: + +```typescript +// Enable for static assets +{ + action: 'set_cache_settings', + action_parameters: { + cache_reserve: { eligible: true, minimum_file_ttl: 86400 }, + edge_ttl: { mode: 'override_origin', default: 86400 }, + cache: true + }, + expression: '(http.request.uri.path matches "\\.(jpg|png|webp|pdf|zip)$")' +} + +// Disable for APIs +{ + action: 'set_cache_settings', + action_parameters: { cache_reserve: { eligible: false } }, + expression: '(http.request.uri.path matches "^/api/")' +} + +// Create via API: PUT to zones/{zone_id}/rulesets/phases/http_request_cache_settings/entrypoint +``` + +## Wrangler Integration + +Cache Reserve works automatically with Workers deployed via Wrangler. No special wrangler.jsonc configuration needed - enable Cache Reserve via Dashboard or API for the zone. + +## See Also + +- [README](./README.md) - Overview and core concepts +- [API Reference](./api.md) - Purging and monitoring APIs +- [Patterns](./patterns.md) - Best practices and optimization +- [Gotchas](./gotchas.md) - Common issues and troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/cache-reserve/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/gotchas.md new file mode 100644 index 0000000..9995cf8 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/gotchas.md @@ -0,0 +1,132 @@ +# Cache Reserve Gotchas + +## Common Errors + +### "Assets Not Being Cached in Cache Reserve" + +**Cause:** Asset is not cacheable, TTL < 10 hours, Content-Length header missing, or blocking headers present (Set-Cookie, Vary: *) +**Solution:** Ensure minimum TTL of 10+ hours (`Cache-Control: public, max-age=36000`), add Content-Length header, remove Set-Cookie header, and set `Vary: Accept-Encoding` (not *) + +### "Range Requests Not Working" (Video Seeking Fails) + +**Cause:** Cache Reserve does **NOT** support range requests (HTTP 206 Partial Content) +**Solution:** Range requests bypass Cache Reserve entirely. For video streaming with seeking: +- Use edge cache only (shorter TTLs) +- Consider R2 with direct access for range-heavy workloads +- Accept that seekable content won't benefit from Cache Reserve persistence + +### "Origin Bandwidth Higher Than Expected" + +**Cause:** Cache Reserve fetches **uncompressed** content from origin, even though it serves compressed to visitors +**Solution:** +- If origin charges by bandwidth, factor in uncompressed transfer costs +- Cache Reserve compresses for visitors automatically (saves visitor bandwidth) +- Compare: origin egress savings vs higher uncompressed fetch costs + +### "Cloudflare Images Not Caching with Cache Reserve" + +**Cause:** Cloudflare Images with `Vary: Accept` header (format negotiation) is incompatible with Cache Reserve +**Solution:** +- Cache Reserve silently skips images with Vary for format negotiation +- Original images (non-transformed) may still be eligible +- Use Cloudflare Images variants or edge cache for transformed images + +### "High Class A Operations Costs" + +**Cause:** Frequent cache misses, short TTLs, or frequent revalidation +**Solution:** Increase TTL for stable content (24+ hours), enable Tiered Cache to reduce direct Cache Reserve misses, or use stale-while-revalidate + +### "Purge Not Working as Expected" + +**Cause:** Purge by tag only triggers revalidation but doesn't remove from Cache Reserve storage +**Solution:** Use purge by URL for immediate removal, or disable Cache Reserve then clear all data for complete removal + +### "O2O (Orange-to-Orange) Assets Not Caching" + +**Cause:** Orange-to-Orange (proxied zone requesting another proxied zone on Cloudflare) bypasses Cache Reserve +**Solution:** +- **What is O2O**: Zone A (proxied) → Zone B (proxied), both on Cloudflare +- **Detection**: Check `cf-cache-status` for `BYPASS` and review request path +- **Workaround**: Use R2 or direct origin access instead of O2O proxy chains + +### "Cache Reserve must be OFF before clearing data" + +**Cause:** Attempting to clear Cache Reserve data while it's still enabled +**Solution:** Disable Cache Reserve first, wait briefly for propagation (5s), then clear data (can take up to 24 hours) + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Minimum TTL | 10 hours (36000 seconds) | Assets with shorter TTL not eligible | +| Default retention | 30 days (2592000 seconds) | Configurable | +| Maximum file size | Same as R2 limits | No practical limit | +| Purge/clear time | Up to 24 hours | Complete propagation time | +| Plan requirement | Paid Cache Reserve or Smart Shield | Not available on free plans | +| Content-Length header | Required | Must be present for eligibility | +| Set-Cookie header | Blocks caching | Must not be present (or use private directive) | +| Vary header | Cannot be * | Can use Vary: Accept-Encoding | +| Image transformations | Variants not eligible | Original images only | +| Range requests | NOT supported | HTTP 206 bypasses Cache Reserve | +| Compression | Fetches uncompressed | Serves compressed to visitors | +| Worker control | Zone-level only | Cannot control per-request | +| O2O requests | Bypassed | Orange-to-Orange not eligible | + +## Additional Resources + +- **Official Docs**: https://developers.cloudflare.com/cache/advanced-configuration/cache-reserve/ +- **API Reference**: https://developers.cloudflare.com/api/resources/cache/subresources/cache_reserve/ +- **Cache Rules**: https://developers.cloudflare.com/cache/how-to/cache-rules/ +- **Workers Cache API**: https://developers.cloudflare.com/workers/runtime-apis/cache/ +- **R2 Documentation**: https://developers.cloudflare.com/r2/ +- **Smart Shield**: https://developers.cloudflare.com/smart-shield/ +- **Tiered Cache**: https://developers.cloudflare.com/cache/how-to/tiered-cache/ + +## Troubleshooting Flowchart + +Asset not caching in Cache Reserve? + +``` +1. Is Cache Reserve enabled for zone? + → No: Enable via Dashboard or API + → Yes: Continue to step 2 + +2. Is Tiered Cache enabled? + → No: Enable Tiered Cache (required!) + → Yes: Continue to step 3 + +3. Does asset have TTL ≥ 10 hours? + → No: Increase via Cache Rules (edge_ttl override) + → Yes: Continue to step 4 + +4. Is Content-Length header present? + → No: Fix origin to include Content-Length + → Yes: Continue to step 5 + +5. Is Set-Cookie header present? + → Yes: Remove Set-Cookie or scope appropriately + → No: Continue to step 6 + +6. Is Vary header set to *? + → Yes: Change to specific value (e.g., Accept-Encoding) + → No: Continue to step 7 + +7. Is this a range request? + → Yes: Range requests bypass Cache Reserve (not supported) + → No: Continue to step 8 + +8. Is this an O2O (Orange-to-Orange) request? + → Yes: O2O bypasses Cache Reserve + → No: Continue to step 9 + +9. Check Logpush CacheReserveUsed field + → Filter logs to see if assets ever hit Cache Reserve + → Verify cf-cache-status header (should be HIT after first request) +``` + +## See Also + +- [README](./README.md) - Overview and core concepts +- [Configuration](./configuration.md) - Setup and Cache Rules +- [API Reference](./api.md) - Purging and monitoring +- [Patterns](./patterns.md) - Best practices and optimization diff --git a/plugins/cloudflare/skills/cloudflare/references/cache-reserve/patterns.md b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/patterns.md new file mode 100644 index 0000000..65f9488 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cache-reserve/patterns.md @@ -0,0 +1,197 @@ +# Cache Reserve Patterns + +## Best Practices + +### 1. Always Enable Tiered Cache + +```typescript +// Cache Reserve is designed for use WITH Tiered Cache +const configuration = { + tieredCache: 'enabled', // Required for optimal performance + cacheReserve: 'enabled', // Works best with Tiered Cache + + hierarchy: [ + 'Lower-Tier Cache (visitor)', + 'Upper-Tier Cache (origin region)', + 'Cache Reserve (persistent)', + 'Origin' + ] +}; +``` + +### 2. Set Appropriate Cache-Control Headers + +```typescript +// Origin response headers for Cache Reserve eligibility +const originHeaders = { + 'Cache-Control': 'public, max-age=86400', // 24hr (minimum 10hr) + 'Content-Length': '1024000', // Required + 'Cache-Tag': 'images,product-123', // Optional: purging + 'ETag': '"abc123"', // Optional: revalidation + // Avoid: 'Set-Cookie' and 'Vary: *' prevent caching +}; +``` + +### 3. Use Cache Rules for Fine-Grained Control + +```typescript +// Different TTLs for different content types +const cacheRules = [ + { + description: 'Long-term cache for immutable assets', + expression: '(http.request.uri.path matches "^/static/.*\\.[a-f0-9]{8}\\.")', + action_parameters: { + cache_reserve: { eligible: true }, + edge_ttl: { mode: 'override_origin', default: 2592000 }, // 30 days + cache: true + } + }, + { + description: 'Moderate cache for regular images', + expression: '(http.request.uri.path matches "\\.(jpg|png|webp)$")', + action_parameters: { + cache_reserve: { eligible: true }, + edge_ttl: { mode: 'override_origin', default: 86400 }, // 24 hours + cache: true + } + }, + { + description: 'Exclude API from Cache Reserve', + expression: '(http.request.uri.path matches "^/api/")', + action_parameters: { cache_reserve: { eligible: false }, cache: false } + } +]; +``` + +### 4. Making Assets Cache Reserve Eligible from Workers + +**Note**: This modifies response headers to meet eligibility criteria but does NOT directly control Cache Reserve storage (which is zone-level automatic). + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const response = await fetch(request); + if (!response.ok) return response; + + const headers = new Headers(response.headers); + headers.set('Cache-Control', 'public, max-age=36000'); // 10hr minimum + headers.delete('Set-Cookie'); // Blocks caching + + // Ensure Content-Length present + if (!headers.has('Content-Length')) { + const blob = await response.blob(); + headers.set('Content-Length', blob.size.toString()); + return new Response(blob, { status: response.status, headers }); + } + + return new Response(response.body, { status: response.status, headers }); + } +}; +``` + +### 5. Hostname Best Practices + +Use Worker's hostname for efficient caching - avoid overriding hostname unnecessarily. + +## Architecture Patterns + +### Multi-Tier Caching + Immutable Assets + +```typescript +// Optimal: L1 (visitor) → L2 (region) → L3 (Cache Reserve) → Origin +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + const isImmutable = /\.[a-f0-9]{8,}\.(js|css|jpg|png|woff2)$/.test(url.pathname); + const response = await fetch(request); + + if (isImmutable) { + const headers = new Headers(response.headers); + headers.set('Cache-Control', 'public, max-age=31536000, immutable'); + return new Response(response.body, { status: response.status, headers }); + } + return response; + } +}; +``` + +## Cost Optimization + +### Cost Calculator + +```typescript +interface CacheReserveEstimate { + avgAssetSizeGB: number; + uniqueAssets: number; + monthlyReads: number; + monthlyWrites: number; + originEgressCostPerGB: number; // e.g., AWS: $0.09/GB +} + +function estimateMonthlyCost(input: CacheReserveEstimate) { + // Cache Reserve pricing + const storageCostPerGBMonth = 0.015; + const classAPerMillion = 4.50; // writes + const classBPerMillion = 0.36; // reads + + // Calculate Cache Reserve costs + const totalStorageGB = input.avgAssetSizeGB * input.uniqueAssets; + const storageCost = totalStorageGB * storageCostPerGBMonth; + const writeCost = (input.monthlyWrites / 1_000_000) * classAPerMillion; + const readCost = (input.monthlyReads / 1_000_000) * classBPerMillion; + + const cacheReserveCost = storageCost + writeCost + readCost; + + // Calculate origin egress cost (what you'd pay without Cache Reserve) + const totalTrafficGB = (input.monthlyReads * input.avgAssetSizeGB); + const originEgressCost = totalTrafficGB * input.originEgressCostPerGB; + + // Savings calculation + const savings = originEgressCost - cacheReserveCost; + const savingsPercent = ((savings / originEgressCost) * 100).toFixed(1); + + return { + cacheReserveCost: `$${cacheReserveCost.toFixed(2)}`, + originEgressCost: `$${originEgressCost.toFixed(2)}`, + monthlySavings: `$${savings.toFixed(2)}`, + savingsPercent: `${savingsPercent}%`, + breakdown: { + storage: `$${storageCost.toFixed(2)}`, + writes: `$${writeCost.toFixed(2)}`, + reads: `$${readCost.toFixed(2)}`, + } + }; +} + +// Example: Media library +const mediaLibrary = estimateMonthlyCost({ + avgAssetSizeGB: 0.005, // 5MB images + uniqueAssets: 10_000, + monthlyReads: 5_000_000, + monthlyWrites: 50_000, + originEgressCostPerGB: 0.09, // AWS S3 +}); + +console.log(mediaLibrary); +// { +// cacheReserveCost: "$9.98", +// originEgressCost: "$25.00", +// monthlySavings: "$15.02", +// savingsPercent: "60.1%", +// breakdown: { storage: "$0.75", writes: "$0.23", reads: "$9.00" } +// } +``` + +### Optimization Guidelines + +- **Set appropriate TTLs**: 10hr minimum, 24hr+ optimal for stable content, 30d max cautiously +- **Cache high-value stable assets**: Images, media, fonts, archives, documentation +- **Exclude frequently changing**: APIs, user-specific content, real-time data +- **Compression note**: Cache Reserve fetches uncompressed from origin, serves compressed to visitors - factor in origin egress costs + +## See Also + +- [README](./README.md) - Overview and core concepts +- [Configuration](./configuration.md) - Setup and Cache Rules +- [API Reference](./api.md) - Purging and monitoring +- [Gotchas](./gotchas.md) - Common issues and troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/containers/README.md b/plugins/cloudflare/skills/cloudflare/references/containers/README.md new file mode 100644 index 0000000..a6c488d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/containers/README.md @@ -0,0 +1,85 @@ +# Cloudflare Containers Skill Reference + +**APPLIES TO: Cloudflare Containers ONLY - NOT general Cloudflare Workers** + +Use when working with Cloudflare Containers: deploying containerized apps on Workers platform, configuring container-enabled Durable Objects, managing container lifecycle, or implementing stateful/stateless container patterns. + +## Beta Status + +⚠️ Containers is currently in **beta**. API may change without notice. No SLA guarantees. Custom instance types added Jan 2026. + +## Core Concepts + +**Container as Durable Object:** Each container is a Durable Object with persistent identity. Accessed via `getByName(id)` or `getRandom()`. + +**Image deployment:** Images pre-fetched globally. Deployments use rolling strategy (not instant like Workers). + +**Lifecycle:** cold start (2-3s) → running → `sleepAfter` timeout → stopped. No autoscaling - manual load balancing via `getRandom()`. + +**Persistent identity, ephemeral disk:** Container ID persists, but disk resets on stop. Use Durable Object storage for persistence. + +## Quick Start + +```typescript +import { Container } from "@cloudflare/containers"; + +export class MyContainer extends Container { + defaultPort = 8080; + sleepAfter = "30m"; +} + +export default { + async fetch(request: Request, env: Env) { + const container = env.MY_CONTAINER.getByName("instance-1"); + await container.startAndWaitForPorts(); + return container.fetch(request); + } +}; +``` + +## Reading Order + +| Task | Files | +|------|-------| +| Setup new container project | README → configuration.md | +| Implement container logic | README → api.md → patterns.md | +| Choose routing pattern | patterns.md (routing section) | +| Debug issues | gotchas.md | +| Production hardening | gotchas.md → patterns.md (lifecycle) | + +## Routing Decision Tree + +**How should requests reach containers?** + +- **Same user/session → same container:** Use `getByName(sessionId)` for session affinity +- **Stateless, spread load:** Use `getRandom()` for load balancing +- **Job per container:** Use `getByName(jobId)` + explicit lifecycle management +- **Single global instance:** Use `getByName("singleton")` + +## When to Use Containers vs Workers + +**Use Containers when:** +- Need stateful, long-lived processes (sessions, WebSockets, games) +- Running existing containerized apps (Node.js, Python, custom binaries) +- Need filesystem access or specific system dependencies +- Per-user/session isolation with dedicated compute + +**Use Workers when:** +- Stateless HTTP handlers +- Sub-millisecond cold starts required +- Auto-scaling to zero critical +- Simple request/response patterns + +## In This Reference + +- **[configuration.md](configuration.md)** - Wrangler config, instance types, Container class properties, environment variables, account limits +- **[api.md](api.md)** - Container class API, startup methods, communication (HTTP/TCP/WebSocket), routing helpers, lifecycle hooks, scheduling, state inspection +- **[patterns.md](patterns.md)** - Routing patterns (session affinity, load balancing, singleton), WebSocket forwarding, graceful shutdown, Workflow/Queue integration +- **[gotchas.md](gotchas.md)** - Critical gotchas (WebSocket, startup methods), common errors with solutions, specific limits, beta caveats + +## See Also + +- [Durable Objects](../durable-objects/) - Containers extend Durable Objects +- [Workflows](../workflows/) - Orchestrate container operations +- [Queues](../queues/) - Trigger containers from queue messages +- [Cloudflare Docs](https://developers.cloudflare.com/containers/) diff --git a/plugins/cloudflare/skills/cloudflare/references/containers/api.md b/plugins/cloudflare/skills/cloudflare/references/containers/api.md new file mode 100644 index 0000000..c41f721 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/containers/api.md @@ -0,0 +1,187 @@ +## Container Class API + +```typescript +import { Container } from "@cloudflare/containers"; + +export class MyContainer extends Container { + defaultPort = 8080; + requiredPorts = [8080]; + sleepAfter = "30m"; + enableInternet = true; + pingEndpoint = "/health"; + envVars = {}; + entrypoint = []; + + onStart() { /* container started */ } + onStop() { /* container stopping */ } + onError(error: Error) { /* container error */ } + onActivityExpired(): boolean { /* timeout, return true to stay alive */ } + async alarm() { /* scheduled task */ } +} +``` + +## Routing + +**getByName(id)** - Named instance for session affinity, per-user state +**getRandom()** - Random instance for load balancing stateless services + +```typescript +const container = env.MY_CONTAINER.getByName("user-123"); +const container = env.MY_CONTAINER.getRandom(); +``` + +## Startup Methods + +### start() - Basic start (8s timeout) + +```typescript +await container.start(); +await container.start({ envVars: { KEY: "value" } }); +``` + +Returns when **process starts**, NOT when ports ready. Use for fire-and-forget. + +### startAndWaitForPorts() - Recommended (20s timeout) + +```typescript +await container.startAndWaitForPorts(); // Uses requiredPorts +await container.startAndWaitForPorts({ ports: [8080, 9090] }); +await container.startAndWaitForPorts({ + ports: [8080], + startOptions: { envVars: { KEY: "value" } } +}); +``` + +Returns when **ports listening**. Use before HTTP/TCP requests. + +**Port resolution:** explicit ports → requiredPorts → defaultPort → port 33 + +### waitForPort() - Wait for specific port + +```typescript +await container.waitForPort(8080); +await container.waitForPort(8080, { timeout: 30000 }); +``` + +## Communication + +### fetch() - HTTP with WebSocket support + +```typescript +// ✅ Supports WebSocket upgrades +const response = await container.fetch(request); +const response = await container.fetch("http://container/api", { + method: "POST", + body: JSON.stringify({ data: "value" }) +}); +``` + +**Use for:** All HTTP, especially WebSocket. + +### containerFetch() - HTTP only (no WebSocket) + +```typescript +// ❌ No WebSocket support +const response = await container.containerFetch(request); +``` + +**⚠️ Critical:** Use `fetch()` for WebSocket, not `containerFetch()`. + +### TCP Connections + +```typescript +const port = this.ctx.container.getTcpPort(8080); +const conn = port.connect(); +await conn.opened; + +if (request.body) await request.body.pipeTo(conn.writable); +return new Response(conn.readable); +``` + +### switchPort() - Change default port + +```typescript +this.switchPort(8081); // Subsequent fetch() uses this port +``` + +## Lifecycle Hooks + +### onStart() + +Called when container process starts (ports may not be ready). Runs in `blockConcurrencyWhile` - no concurrent requests. + +```typescript +onStart() { + console.log("Container starting"); +} +``` + +### onStop() + +Called when SIGTERM received. 15 minutes until SIGKILL. Use for graceful shutdown. + +```typescript +onStop() { + // Save state, close connections, flush logs +} +``` + +### onError() + +Called when container crashes or fails to start. + +```typescript +onError(error: Error) { + console.error("Container error:", error); +} +``` + +### onActivityExpired() + +Called when `sleepAfter` timeout reached. Return `true` to stay alive, `false` to stop. + +```typescript +onActivityExpired(): boolean { + if (this.hasActiveConnections()) return true; // Keep alive + return false; // OK to stop +} +``` + +## Scheduling + +```typescript +export class ScheduledContainer extends Container { + async fetch(request: Request) { + await this.schedule(Date.now() + 60000); // 1 minute + await this.schedule("2026-01-28T00:00:00Z"); // ISO string + return new Response("Scheduled"); + } + + async alarm() { + // Called when schedule fires (SQLite-backed, survives restarts) + } +} +``` + +**⚠️ Don't override `alarm()` directly when using `schedule()` helper.** + +## State Inspection + +### External state check + +```typescript +const state = await container.getState(); +// state.status: "starting" | "running" | "stopping" | "stopped" +``` + +### Internal state check + +```typescript +export class MyContainer extends Container { + async fetch(request: Request) { + if (this.ctx.container.running) { ... } + } +} +``` + +**⚠️ Use `getState()` for external checks, `ctx.container.running` for internal.** diff --git a/plugins/cloudflare/skills/cloudflare/references/containers/configuration.md b/plugins/cloudflare/skills/cloudflare/references/containers/configuration.md new file mode 100644 index 0000000..fd39cc4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/containers/configuration.md @@ -0,0 +1,188 @@ +## Wrangler Configuration + +### Basic Container Config + +```jsonc +{ + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2026-01-10", + "containers": [ + { + "class_name": "MyContainer", + "image": "./Dockerfile", // Path to Dockerfile or directory with Dockerfile + "instance_type": "standard-1", // Predefined or custom (see below) + "max_instances": 10 + } + ], + "durable_objects": { + "bindings": [ + { + "name": "MY_CONTAINER", + "class_name": "MyContainer" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["MyContainer"] // Must use new_sqlite_classes + } + ] +} +``` + +Key config requirements: +- `image` - Path to Dockerfile or directory containing Dockerfile +- `class_name` - Must match Container class export name +- `max_instances` - Max concurrent container instances +- Must configure Durable Objects binding AND migrations + +### Instance Types + +#### Predefined Types + +| Type | vCPU | Memory | Disk | +|------|------|--------|------| +| lite | 1/16 | 256 MiB | 2 GB | +| basic | 1/4 | 1 GiB | 4 GB | +| standard-1 | 1/2 | 4 GiB | 8 GB | +| standard-2 | 1 | 6 GiB | 12 GB | +| standard-3 | 2 | 8 GiB | 16 GB | +| standard-4 | 4 | 12 GiB | 20 GB | + +```jsonc +{ + "containers": [ + { + "class_name": "MyContainer", + "image": "./Dockerfile", + "instance_type": "standard-2" // Use predefined type + } + ] +} +``` + +#### Custom Types (Jan 2026 Feature) + +```jsonc +{ + "containers": [ + { + "class_name": "MyContainer", + "image": "./Dockerfile", + "instance_type_custom": { + "vcpu": 2, // 1-4 vCPU + "memory_mib": 8192, // 512-12288 MiB (up to 12 GiB) + "disk_mib": 16384 // 2048-20480 MiB (up to 20 GB) + } + } + ] +} +``` + +**Custom type constraints:** +- Minimum 3 GiB memory per vCPU +- Maximum 2 GB disk per 1 GiB memory +- Max 4 vCPU, 12 GiB memory, 20 GB disk per container + +### Account Limits + +| Resource | Limit | Notes | +|----------|-------|-------| +| Total memory (all containers) | 400 GiB | Across all running containers | +| Total vCPU (all containers) | 100 | Across all running containers | +| Total disk (all containers) | 2 TB | Across all running containers | +| Image storage per account | 50 GB | Stored container images | + +### Container Class Properties + +```typescript +import { Container } from "@cloudflare/containers"; + +export class MyContainer extends Container { + // Port Configuration + defaultPort = 8080; // Default port for fetch() calls + requiredPorts = [8080, 9090]; // Ports to wait for in startAndWaitForPorts() + + // Lifecycle + sleepAfter = "30m"; // Inactivity timeout (5m, 30m, 2h, etc.) + + // Network + enableInternet = true; // Allow outbound internet access + + // Health Check + pingEndpoint = "/health"; // Health check endpoint path + + // Environment + envVars = { // Environment variables passed to container + NODE_ENV: "production", + LOG_LEVEL: "info" + }; + + // Startup + entrypoint = ["/bin/start.sh"]; // Override image entrypoint (optional) +} +``` + +**Property details:** + +- **`defaultPort`**: Port used when calling `container.fetch()` without explicit port. Falls back to port 33 if not set. + +- **`requiredPorts`**: Array of ports that must be listening before `startAndWaitForPorts()` returns. First port becomes default if `defaultPort` not set. + +- **`sleepAfter`**: Duration string (e.g., "5m", "30m", "2h"). Container stops after this period of inactivity. Timer resets on each request. + +- **`enableInternet`**: Boolean. If `true`, container can make outbound HTTP/TCP requests. + +- **`pingEndpoint`**: Path used for health checks. Should respond with 2xx status. + +- **`envVars`**: Object of environment variables. Merged with runtime-provided vars (see below). + +- **`entrypoint`**: Array of strings. Overrides container image's CMD/ENTRYPOINT. + +### Runtime Environment Variables + +Cloudflare automatically provides these environment variables to containers: + +| Variable | Description | +|----------|-------------| +| `CLOUDFLARE_APPLICATION_ID` | Worker application ID | +| `CLOUDFLARE_COUNTRY_A2` | Two-letter country code of request origin | +| `CLOUDFLARE_LOCATION` | Cloudflare data center location | +| `CLOUDFLARE_REGION` | Region identifier | +| `CLOUDFLARE_DURABLE_OBJECT_ID` | Container's Durable Object ID | + +Custom `envVars` from Container class are merged with these. Custom vars override runtime vars if names conflict. + +### Image Management + +**Distribution model:** Images pre-fetched to all global locations before deployment. Ensures fast cold starts (2-3s typical). + +**Rolling deploys:** Unlike Workers (instant), container deployments roll out gradually. Old versions continue running during rollout. + +**Ephemeral disk:** Container disk is ephemeral and resets on each stop. Use Durable Object storage (`this.ctx.storage`) for persistence. + +## wrangler.toml Format + +```toml +name = "my-worker" +main = "src/index.ts" +compatibility_date = "2026-01-10" + +[[containers]] +class_name = "MyContainer" +image = "./Dockerfile" +instance_type = "standard-2" +max_instances = 10 + +[[durable_objects.bindings]] +name = "MY_CONTAINER" +class_name = "MyContainer" + +[[migrations]] +tag = "v1" +new_sqlite_classes = ["MyContainer"] +``` + +Both `wrangler.jsonc` and `wrangler.toml` are supported. Use `wrangler.jsonc` for comments and better IDE support. diff --git a/plugins/cloudflare/skills/cloudflare/references/containers/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/containers/gotchas.md new file mode 100644 index 0000000..306e8c5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/containers/gotchas.md @@ -0,0 +1,178 @@ +## Critical Gotchas + +### ⚠️ WebSocket: fetch() vs containerFetch() + +**Problem:** WebSocket connections fail silently + +**Cause:** `containerFetch()` doesn't support WebSocket upgrades + +**Fix:** Always use `fetch()` for WebSocket + +```typescript +// ❌ WRONG +return container.containerFetch(request); + +// ✅ CORRECT +return container.fetch(request); +``` + +### ⚠️ startAndWaitForPorts() vs start() + +**Problem:** "connection refused" after `start()` + +**Cause:** `start()` returns when process starts, NOT when ports ready + +**Fix:** Use `startAndWaitForPorts()` before requests + +```typescript +// ❌ WRONG +await container.start(); +return container.fetch(request); + +// ✅ CORRECT +await container.startAndWaitForPorts(); +return container.fetch(request); +``` + +### ⚠️ Activity Timeout on Long Operations + +**Problem:** Container stops during long work + +**Cause:** `sleepAfter` based on request activity, not internal work + +**Fix:** Renew timeout by touching storage + +```typescript +const interval = setInterval(() => { + this.ctx.storage.put("keepalive", Date.now()); +}, 60000); + +try { + await this.doLongWork(data); +} finally { + clearInterval(interval); +} +``` + +### ⚠️ blockConcurrencyWhile for Startup + +**Problem:** Race conditions during initialization + +**Fix:** Use `blockConcurrencyWhile` for atomic initialization + +```typescript +await this.ctx.blockConcurrencyWhile(async () => { + if (!this.initialized) { + await this.startAndWaitForPorts(); + this.initialized = true; + } +}); +``` + +### ⚠️ Lifecycle Hooks Block Requests + +**Problem:** Container unresponsive during `onStart()` + +**Cause:** Hooks run in `blockConcurrencyWhile` - no concurrent requests + +**Fix:** Keep hooks fast, avoid long operations + +### ⚠️ Don't Override alarm() When Using schedule() + +**Problem:** Scheduled tasks don't execute + +**Cause:** `schedule()` uses `alarm()` internally + +**Fix:** Implement `alarm()` to handle scheduled tasks + +## Common Errors + +### "Container start timeout" + +**Cause:** Container took >8s (`start()`) or >20s (`startAndWaitForPorts()`) + +**Solutions:** +- Optimize image (smaller base, fewer layers) +- Check `entrypoint` correct +- Verify app listens on correct ports +- Increase timeout if needed + +### "Port not available" + +**Cause:** Calling `fetch()` before port ready + +**Solution:** Use `startAndWaitForPorts()` + +### "Container memory exceeded" + +**Cause:** Using more memory than instance type allows + +**Solutions:** +- Use larger instance type (standard-2, standard-3, standard-4) +- Optimize app memory usage +- Use custom instance type + +```jsonc +"instance_type_custom": { + "vcpu": 2, + "memory_mib": 8192 +} +``` + +### "Max instances reached" + +**Cause:** All `max_instances` slots in use + +**Solutions:** +- Increase `max_instances` +- Implement proper `sleepAfter` +- Use `getRandom()` for distribution +- Check for instance leaks + +### "No container instance available" + +**Cause:** Account capacity limits reached + +**Solutions:** +- Check account limits +- Review instance types across containers +- Contact Cloudflare support + +## Limits + +| Resource | Limit | Notes | +|----------|-------|-------| +| Cold start | 2-3s | Image pre-fetched globally | +| Graceful shutdown | 15 min | SIGTERM → SIGKILL | +| `start()` timeout | 8s | Process start | +| `startAndWaitForPorts()` timeout | 20s | Port ready | +| Max vCPU per container | 4 | standard-4 or custom | +| Max memory per container | 12 GiB | standard-4 or custom | +| Max disk per container | 20 GB | Ephemeral, resets | +| Account total memory | 400 GiB | All containers | +| Account total vCPU | 100 | All containers | +| Account total disk | 2 TB | All containers | +| Image storage | 50 GB | Per account | +| Disk persistence | None | Use DO storage | + +## Best Practices + +1. **Use `startAndWaitForPorts()` by default** - Prevents port errors +2. **Set appropriate `sleepAfter`** - Balance resources vs cold starts +3. **Use `fetch()` for WebSocket** - Not `containerFetch()` +4. **Design for restarts** - Ephemeral disk, implement graceful shutdown +5. **Monitor resources** - Stay within account limits +6. **Keep hooks fast** - Run in `blockConcurrencyWhile` +7. **Renew activity for long ops** - Touch storage to prevent timeout + +## Beta Caveats + +⚠️ Containers in **beta**: + +- **API may change** without notice +- **No SLA** guarantees +- **Limited regions** initially +- **No autoscaling** - manual via `getRandom()` +- **Rolling deploys** only (not instant like Workers) + +Plan for API changes, test thoroughly before production. diff --git a/plugins/cloudflare/skills/cloudflare/references/containers/patterns.md b/plugins/cloudflare/skills/cloudflare/references/containers/patterns.md new file mode 100644 index 0000000..9204294 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/containers/patterns.md @@ -0,0 +1,202 @@ +## Routing Patterns + +### Session Affinity (Stateful) + +```typescript +export class SessionBackend extends Container { + defaultPort = 3000; + sleepAfter = "30m"; +} + +export default { + async fetch(request: Request, env: Env) { + const sessionId = request.headers.get("X-Session-ID") || crypto.randomUUID(); + const container = env.SESSION_BACKEND.getByName(sessionId); + await container.startAndWaitForPorts(); + return container.fetch(request); + } +}; +``` + +**Use:** User sessions, WebSocket, stateful games, per-user caching. + +### Load Balancing (Stateless) + +```typescript +export default { + async fetch(request: Request, env: Env) { + const container = env.STATELESS_API.getRandom(); + await container.startAndWaitForPorts(); + return container.fetch(request); + } +}; +``` + +**Use:** Stateless HTTP APIs, CPU-intensive work, read-only queries. + +### Singleton Pattern + +```typescript +export default { + async fetch(request: Request, env: Env) { + const container = env.GLOBAL_SERVICE.getByName("singleton"); + await container.startAndWaitForPorts(); + return container.fetch(request); + } +}; +``` + +**Use:** Global cache, centralized coordinator, single source of truth. + +## WebSocket Forwarding + +```typescript +export default { + async fetch(request: Request, env: Env) { + if (request.headers.get("Upgrade") === "websocket") { + const sessionId = request.headers.get("X-Session-ID") || crypto.randomUUID(); + const container = env.WS_BACKEND.getByName(sessionId); + await container.startAndWaitForPorts(); + + // ⚠️ MUST use fetch(), not containerFetch() + return container.fetch(request); + } + return new Response("Not a WebSocket request", { status: 400 }); + } +}; +``` + +**⚠️ Critical:** Always use `fetch()` for WebSocket. + +## Graceful Shutdown + +```typescript +export class GracefulContainer extends Container { + private connections = new Set(); + + onStop() { + // SIGTERM received, 15 minutes until SIGKILL + for (const ws of this.connections) { + ws.close(1001, "Server shutting down"); + } + this.ctx.storage.put("shutdown-time", Date.now()); + } + + onActivityExpired(): boolean { + return this.connections.size > 0; // Keep alive if connections + } +} +``` + +## Concurrent Request Handling + +```typescript +export class SafeContainer extends Container { + private initialized = false; + + async fetch(request: Request) { + await this.ctx.blockConcurrencyWhile(async () => { + if (!this.initialized) { + await this.startAndWaitForPorts(); + this.initialized = true; + } + }); + return super.fetch(request); + } +} +``` + +**Use:** One-time initialization, preventing concurrent startup. + +## Activity Timeout Renewal + +```typescript +export class LongRunningContainer extends Container { + sleepAfter = "5m"; + + async processLongJob(data: unknown) { + const interval = setInterval(() => { + this.ctx.storage.put("keepalive", Date.now()); + }, 60000); + + try { + await this.doLongWork(data); + } finally { + clearInterval(interval); + } + } +} +``` + +**Use:** Long operations exceeding `sleepAfter`. + +## Multiple Port Routing + +```typescript +export class MultiPortContainer extends Container { + requiredPorts = [8080, 8081, 9090]; + + async fetch(request: Request) { + const path = new URL(request.url).pathname; + if (path.startsWith("/grpc")) this.switchPort(8081); + else if (path.startsWith("/metrics")) this.switchPort(9090); + return super.fetch(request); + } +} +``` + +**Use:** Multi-protocol services (HTTP + gRPC), separate metrics endpoints. + +## Workflow Integration + +```typescript +import { WorkflowEntrypoint } from "cloudflare:workers"; + +export class ProcessingWorkflow extends WorkflowEntrypoint { + async run(event, step) { + const container = this.env.PROCESSOR.getByName(event.payload.jobId); + + await step.do("start", async () => { + await container.startAndWaitForPorts(); + }); + + const result = await step.do("process", async () => { + return container.fetch("/process", { + method: "POST", + body: JSON.stringify(event.payload.data) + }).then(r => r.json()); + }); + + return result; + } +} +``` + +**Use:** Orchestrating multi-step container operations, durable execution. + +## Queue Consumer Integration + +```typescript +export default { + async queue(batch, env) { + for (const msg of batch.messages) { + try { + const container = env.PROCESSOR.getByName(msg.body.jobId); + await container.startAndWaitForPorts(); + + const response = await container.fetch("/process", { + method: "POST", + body: JSON.stringify(msg.body) + }); + + response.ok ? msg.ack() : msg.retry(); + } catch (err) { + console.error("Queue processing error:", err); + msg.retry(); + } + } + } +}; +``` + +**Use:** Asynchronous job processing, batch operations, event-driven execution. diff --git a/plugins/cloudflare/skills/cloudflare/references/cron-triggers/README.md b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/README.md new file mode 100644 index 0000000..67c00f8 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/README.md @@ -0,0 +1,99 @@ +# Cloudflare Cron Triggers + +Schedule Workers execution using cron expressions. Runs on Cloudflare's global network during underutilized periods. + +## Key Features + +- **UTC-only execution** - All schedules run on UTC time +- **5-field cron syntax** - Quartz scheduler extensions (L, W, #) +- **Global propagation** - 15min deployment delay +- **At-least-once delivery** - Rare duplicate executions possible +- **Workflow integration** - Trigger long-running multi-step tasks +- **Green Compute** - Optional carbon-aware scheduling during low-carbon periods + +## Cron Syntax + +``` + ┌─────────── minute (0-59) + │ ┌───────── hour (0-23) + │ │ ┌─────── day of month (1-31) + │ │ │ ┌───── month (1-12, JAN-DEC) + │ │ │ │ ┌─── day of week (1-7, SUN-SAT, 1=Sunday) + * * * * * +``` + +**Special chars:** `*` (any), `,` (list), `-` (range), `/` (step), `L` (last), `W` (weekday), `#` (nth) + +## Common Schedules + +```bash +*/5 * * * * # Every 5 minutes +0 * * * * # Hourly +0 2 * * * # Daily 2am UTC (off-peak) +0 9 * * MON-FRI # Weekdays 9am UTC +0 0 1 * * # Monthly 1st midnight UTC +0 9 L * * # Last day of month 9am UTC +0 10 * * MON#2 # 2nd Monday 10am UTC +*/10 9-17 * * MON-FRI # Every 10min, 9am-5pm weekdays +``` + +## Quick Start + +**wrangler.jsonc:** +```jsonc +{ + "name": "my-cron-worker", + "triggers": { + "crons": ["*/5 * * * *", "0 2 * * *"] + } +} +``` + +**Handler:** +```typescript +export default { + async scheduled( + controller: ScheduledController, + env: Env, + ctx: ExecutionContext, + ): Promise { + console.log("Cron:", controller.cron); + console.log("Time:", new Date(controller.scheduledTime)); + + ctx.waitUntil(asyncTask(env)); // Non-blocking + }, +}; +``` + +**Test locally:** +```bash +npx wrangler dev +curl "http://localhost:8787/__scheduled?cron=*/5+*+*+*+*" +``` + +## Limits + +- **Free:** 3 triggers/worker, 10ms CPU +- **Paid:** Unlimited triggers, 50ms CPU +- **Propagation:** 15min global deployment +- **Timezone:** UTC only + +## Reading Order + +**New to cron triggers?** Start here: +1. This README - Overview and quick start +2. [configuration.md](./configuration.md) - Set up your first cron trigger +3. [api.md](./api.md) - Understand the handler API +4. [patterns.md](./patterns.md) - Common use cases and examples + +**Troubleshooting?** Jump to [gotchas.md](./gotchas.md) + +## In This Reference +- [configuration.md](./configuration.md) - wrangler config, env-specific schedules, Green Compute +- [api.md](./api.md) - ScheduledController, noRetry(), waitUntil, testing patterns +- [patterns.md](./patterns.md) - Use cases, monitoring, queue integration, Durable Objects +- [gotchas.md](./gotchas.md) - Timezone issues, idempotency, security, testing + +## See Also +- [workflows](../workflows/) - Alternative for long-running scheduled tasks +- [workers](../workers/) - Worker runtime documentation diff --git a/plugins/cloudflare/skills/cloudflare/references/cron-triggers/api.md b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/api.md new file mode 100644 index 0000000..b0242d7 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/api.md @@ -0,0 +1,196 @@ +# Cron Triggers API + +## Basic Handler + +```typescript +export default { + async scheduled(controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise { + console.log("Cron executed:", new Date(controller.scheduledTime)); + }, +}; +``` + +**JavaScript:** Same signature without types +**Python:** `class Default(WorkerEntrypoint): async def scheduled(self, controller, env, ctx)` + +## ScheduledController + +```typescript +interface ScheduledController { + scheduledTime: number; // Unix ms when scheduled to run + cron: string; // Expression that triggered (e.g., "*/5 * * * *") + type: string; // Always "scheduled" + noRetry(): void; // Prevent automatic retry on failure +} +``` + +**Prevent retry on failure:** +```typescript +export default { + async scheduled(controller, env, ctx) { + try { + await riskyOperation(env); + } catch (error) { + // Don't retry - failure is expected/acceptable + controller.noRetry(); + console.error("Operation failed, not retrying:", error); + } + }, +}; +``` + +**When to use noRetry():** +- External API failures outside your control (avoid hammering failed services) +- Rate limit errors (retry would fail again immediately) +- Duplicate execution detected (idempotency check failed) +- Non-critical operations where skip is acceptable (analytics, caching) +- Validation errors that won't resolve on retry + +## Handler Parameters + +**`controller: ScheduledController`** +- Access cron expression and scheduled time + +**`env: Env`** +- All bindings: KV, R2, D1, secrets, service bindings + +**`ctx: ExecutionContext`** +- `ctx.waitUntil(promise)` - Extend execution for async tasks (logging, cleanup, external APIs) +- First `waitUntil` failure recorded in Cron Events + +## Multiple Schedules + +```typescript +export default { + async scheduled(controller, env, ctx) { + switch (controller.cron) { + case "*/3 * * * *": ctx.waitUntil(updateRecentData(env)); break; + case "0 * * * *": ctx.waitUntil(processHourlyAggregation(env)); break; + case "0 2 * * *": ctx.waitUntil(performDailyMaintenance(env)); break; + default: console.warn(`Unhandled: ${controller.cron}`); + } + }, +}; +``` + +## ctx.waitUntil Usage + +```typescript +export default { + async scheduled(controller, env, ctx) { + const data = await fetchCriticalData(); // Critical path + + // Non-blocking background tasks + ctx.waitUntil(Promise.all([ + logToAnalytics(data), + cleanupOldRecords(env.DB), + notifyWebhook(env.WEBHOOK_URL, data), + ])); + }, +}; +``` + +## Workflow Integration + +```typescript +import { WorkflowEntrypoint } from "cloudflare:workers"; + +export class DataProcessingWorkflow extends WorkflowEntrypoint { + async run(event, step) { + const data = await step.do("fetch-data", () => fetchLargeDataset()); + const processed = await step.do("process-data", () => processDataset(data)); + await step.do("store-results", () => storeResults(processed)); + } +} + +export default { + async scheduled(controller, env, ctx) { + const instance = await env.MY_WORKFLOW.create({ + params: { scheduledTime: controller.scheduledTime, cron: controller.cron }, + }); + console.log(`Started workflow: ${instance.id}`); + }, +}; +``` + +## Testing Handler + +**Local development (/__scheduled endpoint):** +```bash +# Start dev server +npx wrangler dev + +# Trigger any cron +curl "http://localhost:8787/__scheduled?cron=*/5+*+*+*+*" + +# Trigger specific cron with custom time +curl "http://localhost:8787/__scheduled?cron=0+2+*+*+*&scheduledTime=1704067200000" +``` + +**Query parameters:** +- `cron` - Required. URL-encoded cron expression (use `+` for spaces) +- `scheduledTime` - Optional. Unix timestamp in milliseconds (defaults to current time) + +**Production security:** The `/__scheduled` endpoint is available in production and can be triggered by anyone. Block it or implement authentication - see [gotchas.md](./gotchas.md#security-concerns) + +**Unit testing (Vitest):** +```typescript +// test/scheduled.test.ts +import { describe, it, expect } from "vitest"; +import { env } from "cloudflare:test"; +import worker from "../src/index"; + +describe("Scheduled Handler", () => { + it("processes scheduled event", async () => { + const controller = { scheduledTime: Date.now(), cron: "*/5 * * * *", type: "scheduled" as const, noRetry: () => {} }; + const ctx = { waitUntil: (p: Promise) => p, passThroughOnException: () => {} }; + await worker.scheduled(controller, env, ctx); + expect(await env.MY_KV.get("last_run")).toBeDefined(); + }); + + it("handles multiple crons", async () => { + const ctx = { waitUntil: () => {}, passThroughOnException: () => {} }; + await worker.scheduled({ scheduledTime: Date.now(), cron: "*/5 * * * *", type: "scheduled", noRetry: () => {} }, env, ctx); + expect(await env.MY_KV.get("last_type")).toBe("frequent"); + }); +}); +``` + +## Error Handling + +**Automatic retries:** +- Failed cron executions are retried automatically unless `noRetry()` is called +- Retry happens after a delay (typically minutes) +- Only first `waitUntil()` failure is recorded in Cron Events + +**Best practices:** +```typescript +export default { + async scheduled(controller, env, ctx) { + try { + await criticalOperation(env); + } catch (error) { + // Log error details + console.error("Cron failed:", { + cron: controller.cron, + scheduledTime: controller.scheduledTime, + error: error.message, + stack: error.stack, + }); + + // Decide: retry or skip + if (error.message.includes("rate limit")) { + controller.noRetry(); // Skip retry for rate limits + } + // Otherwise allow automatic retry + throw error; + } + }, +}; +``` + +## See Also + +- [README.md](./README.md) - Overview +- [patterns.md](./patterns.md) - Use cases, examples +- [gotchas.md](./gotchas.md) - Common errors, testing issues diff --git a/plugins/cloudflare/skills/cloudflare/references/cron-triggers/configuration.md b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/configuration.md new file mode 100644 index 0000000..b584369 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/configuration.md @@ -0,0 +1,180 @@ +# Cron Triggers Configuration + +## wrangler.jsonc + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-cron-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + + "triggers": { + "crons": [ + "*/5 * * * *", // Every 5 minutes + "0 */2 * * *", // Every 2 hours + "0 9 * * MON-FRI", // Weekdays at 9am UTC + "0 2 1 * *" // Monthly on 1st at 2am UTC + ] + } +} +``` + +## Green Compute (Beta) + +Schedule crons during low-carbon periods for carbon-aware execution: + +```jsonc +{ + "name": "eco-cron-worker", + "triggers": { + "crons": ["0 2 * * *"] + }, + "placement": { + "mode": "smart" // Runs during low-carbon periods + } +} +``` + +**Modes:** +- `"smart"` - Carbon-aware scheduling (may delay up to 24h for optimal window) +- Default (no placement config) - Standard scheduling (no delay) + +**How it works:** +- Cloudflare delays execution until grid carbon intensity is lower +- Maximum delay: 24 hours from scheduled time +- Ideal for batch jobs with flexible timing requirements + +**Use cases:** +- Nightly data processing and ETL pipelines +- Weekly/monthly report generation +- Database backups and maintenance +- Analytics aggregation +- ML model training + +**Not suitable for:** +- Time-sensitive operations (SLA requirements) +- User-facing features requiring immediate execution +- Real-time monitoring and alerting +- Compliance tasks with strict time windows + +## Environment-Specific Schedules + +```jsonc +{ + "name": "my-cron-worker", + "triggers": { + "crons": ["0 */6 * * *"] // Prod: every 6 hours + }, + "env": { + "staging": { + "triggers": { + "crons": ["*/15 * * * *"] // Staging: every 15min + } + }, + "dev": { + "triggers": { + "crons": ["*/5 * * * *"] // Dev: every 5min + } + } + } +} +``` + +## Schedule Format + +**Structure:** `minute hour day-of-month month day-of-week` + +**Special chars:** `*` (any), `,` (list), `-` (range), `/` (step), `L` (last), `W` (weekday), `#` (nth) + +## Managing Triggers + +**Remove all:** `"triggers": { "crons": [] }` +**Preserve existing:** Omit `"triggers"` field entirely + +## Deployment + +```bash +# Deploy with config crons +npx wrangler deploy + +# Deploy specific environment +npx wrangler deploy --env production + +# View deployments +npx wrangler deployments list +``` + +**⚠️ Changes take up to 15 minutes to propagate globally** + +## API Management + +**Get triggers:** +```bash +curl "https://api.cloudflare.com/client/v4/accounts/{account_id}/workers/scripts/{script_name}/schedules" \ + -H "Authorization: Bearer {api_token}" +``` + +**Update triggers:** +```bash +curl -X PUT "https://api.cloudflare.com/client/v4/accounts/{account_id}/workers/scripts/{script_name}/schedules" \ + -H "Authorization: Bearer {api_token}" \ + -H "Content-Type: application/json" \ + -d '{"crons": ["*/5 * * * *", "0 2 * * *"]}' +``` + +**Delete all:** +```bash +curl -X PUT "https://api.cloudflare.com/client/v4/accounts/{account_id}/workers/scripts/{script_name}/schedules" \ + -H "Authorization: Bearer {api_token}" \ + -H "Content-Type: application/json" \ + -d '{"crons": []}' +``` + +## Combining Multiple Workers + +For complex schedules, use multiple workers: + +```jsonc +// worker-frequent.jsonc +{ + "name": "data-sync-frequent", + "triggers": { "crons": ["*/5 * * * *"] } +} + +// worker-daily.jsonc +{ + "name": "reports-daily", + "triggers": { "crons": ["0 2 * * *"] }, + "placement": { "mode": "smart" } +} + +// worker-weekly.jsonc +{ + "name": "cleanup-weekly", + "triggers": { "crons": ["0 3 * * SUN"] } +} +``` + +**Benefits:** +- Separate CPU limits per worker +- Independent error isolation +- Different Green Compute policies +- Easier to maintain and debug + +## Validation + +**Test cron syntax:** +- [crontab.guru](https://crontab.guru/) - Interactive validator +- Wrangler validates on deploy but won't catch logic errors + +**Common mistakes:** +- `0 0 * * *` runs daily at midnight UTC, not your local timezone +- `*/60 * * * *` is invalid (use `0 * * * *` for hourly) +- `0 2 31 * *` only runs on months with 31 days + +## See Also + +- [README.md](./README.md) - Overview, quick start +- [api.md](./api.md) - Handler implementation +- [patterns.md](./patterns.md) - Multi-cron routing examples diff --git a/plugins/cloudflare/skills/cloudflare/references/cron-triggers/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/gotchas.md new file mode 100644 index 0000000..5906c3a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/gotchas.md @@ -0,0 +1,199 @@ +# Cron Triggers Gotchas + +## Common Errors + +### "Timezone Issues" + +**Problem:** Cron runs at wrong time relative to local timezone +**Cause:** All crons execute in UTC, no local timezone support +**Solution:** Convert local time to UTC manually + +**Conversion formula:** `utcHour = (localHour - utcOffset + 24) % 24` + +**Examples:** +- 9am PST (UTC-8) → `(9 - (-8) + 24) % 24 = 17` → `0 17 * * *` +- 2am EST (UTC-5) → `(2 - (-5) + 24) % 24 = 7` → `0 7 * * *` +- 6pm JST (UTC+9) → `(18 - 9 + 24) % 24 = 33 % 24 = 9` → `0 9 * * *` + +**Daylight Saving Time:** Adjust manually when DST changes, or schedule at times unaffected by DST (e.g., 2am-4am local time usually safe) + +### "Cron Not Executing" + +**Cause:** Missing `scheduled()` export, invalid syntax, propagation delay (<15min), or plan limits +**Solution:** Verify export exists, validate at crontab.guru, wait 15+ min after deploy, check plan limits + +### "Duplicate Executions" + +**Cause:** At-least-once delivery +**Solution:** Track execution IDs in KV - see idempotency pattern below + +### "Execution Failures" + +**Cause:** CPU exceeded, unhandled exceptions, network timeouts, binding errors +**Solution:** Use try-catch, AbortController timeouts, `ctx.waitUntil()` for long ops, or Workflows for heavy tasks + +### "Local Testing Not Working" + +**Problem:** `/__scheduled` endpoint returns 404 or doesn't trigger handler +**Cause:** Missing `scheduled()` export, wrangler not running, or incorrect endpoint format +**Solution:** + +1. Verify `scheduled()` is exported: +```typescript +export default { + async scheduled(controller, env, ctx) { + console.log("Cron triggered"); + }, +}; +``` + +2. Start dev server: +```bash +npx wrangler dev +``` + +3. Use correct endpoint format (URL-encode spaces as `+`): +```bash +# Correct +curl "http://localhost:8787/__scheduled?cron=*/5+*+*+*+*" + +# Wrong (will fail) +curl "http://localhost:8787/__scheduled?cron=*/5 * * * *" +``` + +4. Update Wrangler if outdated: +```bash +npm install -g wrangler@latest +``` + +### "waitUntil() Tasks Not Completing" + +**Problem:** Background tasks in `ctx.waitUntil()` fail silently or don't execute +**Cause:** Promises rejected without error handling, or handler returns before promise settles +**Solution:** Always await or handle errors in waitUntil promises: + +```typescript +export default { + async scheduled(controller, env, ctx) { + // BAD: Silent failures + ctx.waitUntil(riskyOperation()); + + // GOOD: Explicit error handling + ctx.waitUntil( + riskyOperation().catch(err => { + console.error("Background task failed:", err); + return logError(err, env); + }) + ); + }, +}; +``` + +### "Idempotency Issues" + +**Problem:** At-least-once delivery causes duplicate side effects (double charges, duplicate emails) +**Cause:** No deduplication mechanism +**Solution:** Use KV to track execution IDs: + +```typescript +export default { + async scheduled(controller, env, ctx) { + const executionId = `${controller.cron}-${controller.scheduledTime}`; + const existing = await env.EXECUTIONS.get(executionId); + + if (existing) { + console.log("Already executed, skipping"); + controller.noRetry(); + return; + } + + await env.EXECUTIONS.put(executionId, "1", { expirationTtl: 86400 }); // 24h TTL + await performIdempotentOperation(env); + }, +}; +``` + +### "Security Concerns" + +**Problem:** `__scheduled` endpoint exposed in production allows unauthorized cron triggering +**Cause:** Testing endpoint available in deployed Workers +**Solution:** Block `__scheduled` in production: + +```typescript +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + + // Block __scheduled in production + if (url.pathname === "/__scheduled" && env.ENVIRONMENT === "production") { + return new Response("Not Found", { status: 404 }); + } + + return handleRequest(request, env, ctx); + }, + + async scheduled(controller, env, ctx) { + // Your cron logic + }, +}; +``` + +**Also:** Use `env.API_KEY` for secrets (never hardcode) + +**Alternative:** Add middleware to verify request origin: +```typescript +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + + if (url.pathname === "/__scheduled") { + // Check Cloudflare headers to verify internal request + const cfRay = request.headers.get("cf-ray"); + if (!cfRay && env.ENVIRONMENT === "production") { + return new Response("Not Found", { status: 404 }); + } + } + + return handleRequest(request, env, ctx); + }, + + async scheduled(controller, env, ctx) { + // Your cron logic + }, +}; +``` + +## Limits & Quotas + +| Limit | Free | Paid | Notes | +|-------|------|------|-------| +| Triggers per Worker | 3 | Unlimited | Maximum cron schedules per Worker | +| CPU time | 10ms | 50ms | May need `ctx.waitUntil()` or Workflows | +| Execution guarantee | At-least-once | At-least-once | Duplicates possible - use idempotency | +| Propagation delay | Up to 15 minutes | Up to 15 minutes | Time for changes to take effect globally | +| Min interval | 1 minute | 1 minute | Cannot schedule more frequently | +| Cron accuracy | ±1 minute | ±1 minute | Execution may drift slightly | + +## Testing Best Practices + +**Unit tests:** +- Mock `ScheduledController`, `ExecutionContext`, and bindings +- Test each cron expression separately +- Verify `noRetry()` is called when expected +- Use Vitest with `@cloudflare/vitest-pool-workers` for realistic env + +**Integration tests:** +- Test via `/__scheduled` endpoint in dev environment +- Verify idempotency logic with duplicate `scheduledTime` values +- Test error handling and retry behavior + +**Production:** Start with long intervals (`*/30 * * * *`), monitor Cron Events for 24h, set up alerts before reducing interval + +## Resources + +- [Cron Triggers Docs](https://developers.cloudflare.com/workers/configuration/cron-triggers/) +- [Scheduled Handler API](https://developers.cloudflare.com/workers/runtime-apis/handlers/scheduled/) +- [Cloudflare Workflows](https://developers.cloudflare.com/workflows/) +- [Workers Limits](https://developers.cloudflare.com/workers/platform/limits/) +- [Crontab Guru](https://crontab.guru/) - Validator +- [Vitest Pool Workers](https://github.com/cloudflare/workers-sdk/tree/main/fixtures/vitest-pool-workers-examples) diff --git a/plugins/cloudflare/skills/cloudflare/references/cron-triggers/patterns.md b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/patterns.md new file mode 100644 index 0000000..a1f1823 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/cron-triggers/patterns.md @@ -0,0 +1,190 @@ +# Cron Triggers Patterns + +## API Data Sync + +```typescript +export default { + async scheduled(controller, env, ctx) { + const response = await fetch("https://api.example.com/data", {headers: { "Authorization": `Bearer ${env.API_KEY}` }}); + if (!response.ok) throw new Error(`API error: ${response.status}`); + ctx.waitUntil(env.MY_KV.put("cached_data", JSON.stringify(await response.json()), {expirationTtl: 3600})); + }, +}; +``` + +## Database Cleanup + +```typescript +export default { + async scheduled(controller, env, ctx) { + const result = await env.DB.prepare(`DELETE FROM sessions WHERE expires_at < datetime('now')`).run(); + console.log(`Deleted ${result.meta.changes} expired sessions`); + ctx.waitUntil(env.DB.prepare("VACUUM").run()); + }, +}; +``` + +## Report Generation + +```typescript +export default { + async scheduled(controller, env, ctx) { + const startOfWeek = new Date(); startOfWeek.setDate(startOfWeek.getDate() - 7); + const { results } = await env.DB.prepare(`SELECT date, revenue, orders FROM daily_stats WHERE date >= ? ORDER BY date`).bind(startOfWeek.toISOString()).all(); + const report = {period: "weekly", totalRevenue: results.reduce((sum, d) => sum + d.revenue, 0), totalOrders: results.reduce((sum, d) => sum + d.orders, 0), dailyBreakdown: results}; + const reportKey = `reports/weekly-${Date.now()}.json`; + await env.REPORTS_BUCKET.put(reportKey, JSON.stringify(report)); + ctx.waitUntil(env.SEND_EMAIL.fetch("https://example.com/send", {method: "POST", body: JSON.stringify({to: "team@example.com", subject: "Weekly Report", reportUrl: `https://reports.example.com/${reportKey}`})})); + }, +}; +``` + +## Health Checks + +```typescript +export default { + async scheduled(controller, env, ctx) { + const services = [{name: "API", url: "https://api.example.com/health"}, {name: "CDN", url: "https://cdn.example.com/health"}]; + const checks = await Promise.all(services.map(async (service) => { + const start = Date.now(); + try { + const response = await fetch(service.url, { signal: AbortSignal.timeout(5000) }); + return {name: service.name, status: response.ok ? "up" : "down", responseTime: Date.now() - start}; + } catch (error) { + return {name: service.name, status: "down", responseTime: Date.now() - start, error: error.message}; + } + })); + ctx.waitUntil(env.STATUS_KV.put("health_status", JSON.stringify(checks))); + const failures = checks.filter(c => c.status === "down"); + if (failures.length > 0) ctx.waitUntil(fetch(env.ALERT_WEBHOOK, {method: "POST", body: JSON.stringify({text: `${failures.length} service(s) down: ${failures.map(f => f.name).join(", ")}`})})); + }, +}; +``` + +## Batch Processing (Rate-Limited) + +```typescript +export default { + async scheduled(controller, env, ctx) { + const queueData = await env.QUEUE_KV.get("pending_items", "json"); + if (!queueData || queueData.length === 0) return; + const batch = queueData.slice(0, 100); + const results = await Promise.allSettled(batch.map(item => fetch("https://api.example.com/process", {method: "POST", headers: {"Authorization": `Bearer ${env.API_KEY}`, "Content-Type": "application/json"}, body: JSON.stringify(item)}))); + console.log(`Processed ${results.filter(r => r.status === "fulfilled").length}/${batch.length} items`); + ctx.waitUntil(env.QUEUE_KV.put("pending_items", JSON.stringify(queueData.slice(100)))); + }, +}; +``` + +## Queue Integration + +```typescript +export default { + async scheduled(controller, env, ctx) { + const batch = await env.MY_QUEUE.receive({ batchSize: 100 }); + const results = await Promise.allSettled(batch.messages.map(async (msg) => { + await processMessage(msg.body, env); + await msg.ack(); + })); + console.log(`Processed ${results.filter(r => r.status === "fulfilled").length}/${batch.messages.length}`); + }, +}; +``` + +## Monitoring & Observability + +```typescript +export default { + async scheduled(controller, env, ctx) { + const startTime = Date.now(); + const meta = { cron: controller.cron, scheduledTime: controller.scheduledTime }; + console.log("[START]", meta); + try { + const result = await performTask(env); + console.log("[SUCCESS]", { ...meta, duration: Date.now() - startTime, count: result.count }); + ctx.waitUntil(env.METRICS.put(`cron:${controller.scheduledTime}`, JSON.stringify({ ...meta, status: "success" }), { expirationTtl: 2592000 })); + } catch (error) { + console.error("[ERROR]", { ...meta, duration: Date.now() - startTime, error: error.message }); + ctx.waitUntil(fetch(env.ALERT_WEBHOOK, { method: "POST", body: JSON.stringify({ text: `Cron failed: ${controller.cron}`, error: error.message }) })); + throw error; + } + }, +}; +``` + +**View logs:** `npx wrangler tail` or Dashboard → Workers & Pages → Worker → Logs + +## Durable Objects Coordination + +```typescript +export default { + async scheduled(controller, env, ctx) { + const stub = env.COORDINATOR.get(env.COORDINATOR.idFromName("cron-lock")); + const acquired = await stub.tryAcquireLock(controller.scheduledTime); + if (!acquired) { + controller.noRetry(); + return; + } + try { + await performTask(env); + } finally { + await stub.releaseLock(); + } + }, +}; +``` + +## Python Handler + +```python +from workers import WorkerEntrypoint + +class Default(WorkerEntrypoint): + async def scheduled(self, controller, env, ctx): + data = await env.MY_KV.get("key") + ctx.waitUntil(env.DB.execute("DELETE FROM logs WHERE created_at < datetime('now', '-7 days')")) +``` + +## Testing Patterns + +**Local testing with /__scheduled:** +```bash +# Start dev server +npx wrangler dev + +# Test specific cron +curl "http://localhost:8787/__scheduled?cron=*/5+*+*+*+*" + +# Test with specific time +curl "http://localhost:8787/__scheduled?cron=0+2+*+*+*&scheduledTime=1704067200000" +``` + +**Unit tests:** +```typescript +// test/scheduled.test.ts +import { describe, it, expect, vi } from "vitest"; +import { env } from "cloudflare:test"; +import worker from "../src/index"; + +describe("Scheduled Handler", () => { + it("executes cron", async () => { + const controller = { scheduledTime: Date.now(), cron: "*/5 * * * *", type: "scheduled" as const, noRetry: vi.fn() }; + const ctx = { waitUntil: vi.fn(), passThroughOnException: vi.fn() }; + await worker.scheduled(controller, env, ctx); + expect(await env.MY_KV.get("last_run")).toBeDefined(); + }); + + it("calls noRetry on duplicate", async () => { + const controller = { scheduledTime: 1704067200000, cron: "0 2 * * *", type: "scheduled" as const, noRetry: vi.fn() }; + await env.EXECUTIONS.put("0 2 * * *-1704067200000", "1"); + await worker.scheduled(controller, env, { waitUntil: vi.fn(), passThroughOnException: vi.fn() }); + expect(controller.noRetry).toHaveBeenCalled(); + }); +}); +``` + +## See Also + +- [README.md](./README.md) - Overview +- [api.md](./api.md) - Handler implementation +- [gotchas.md](./gotchas.md) - Troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/d1/README.md b/plugins/cloudflare/skills/cloudflare/references/d1/README.md new file mode 100644 index 0000000..e40d44c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/d1/README.md @@ -0,0 +1,133 @@ +# Cloudflare D1 Database + +Expert guidance for Cloudflare D1, a serverless SQLite database designed for horizontal scale-out across multiple databases. + +## Overview + +D1 is Cloudflare's managed, serverless database with: +- SQLite SQL semantics and compatibility +- Built-in disaster recovery via Time Travel (30-day point-in-time recovery) +- Horizontal scale-out architecture (10 GB per database) +- Worker and HTTP API access +- Pricing based on query and storage costs only + +**Architecture Philosophy**: D1 is optimized for per-user, per-tenant, or per-entity database patterns rather than single large databases. + +## Quick Start + +```bash +# Create database +wrangler d1 create + +# Execute migration +wrangler d1 migrations apply --remote + +# Local development +wrangler dev +``` + +## Core Query Methods + +```typescript +// .all() - Returns all rows; .first() - First row or null; .first(col) - Single column value +// .run() - INSERT/UPDATE/DELETE; .raw() - Array of arrays (efficient) +const { results, success, meta } = await env.DB.prepare('SELECT * FROM users WHERE active = ?').bind(true).all(); +const user = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); +``` + +## Batch Operations + +```typescript +// Multiple queries in single round trip (atomic transaction) +const results = await env.DB.batch([ + env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(1), + env.DB.prepare('SELECT * FROM posts WHERE author_id = ?').bind(1), + env.DB.prepare('UPDATE users SET last_access = ? WHERE id = ?').bind(Date.now(), 1) +]); +``` + +## Sessions API (Paid Plans) + +```typescript +// Create long-running session for analytics/migrations (up to 15 minutes) +const session = env.DB.withSession(); +try { + await session.prepare('CREATE INDEX idx_heavy ON large_table(column)').run(); + await session.prepare('ANALYZE').run(); +} finally { + session.close(); // Always close to release resources +} +``` + +## Read Replication (Paid Plans) + +```typescript +// Read from nearest replica for lower latency (automatic failover) +const user = await env.DB_REPLICA.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); + +// Writes always go to primary +await env.DB.prepare('UPDATE users SET last_login = ? WHERE id = ?').bind(Date.now(), userId).run(); +``` + +## Platform Limits + +| Limit | Free Tier | Paid Plans | +|-------|-----------|------------| +| Database size | 500 MB | 10 GB per database | +| Row size | 1 MB max | 1 MB max | +| Query timeout | 30 seconds | 30 seconds | +| Batch size | 1,000 statements | 10,000 statements | +| Time Travel retention | 7 days | 30 days | +| Read replicas | Not available | Yes (paid add-on) | + +**Pricing**: $5/month per database beyond free tier + $0.001 per 1K reads + $1 per 1M writes + $0.75/GB storage/month + +## CLI Commands + +```bash +# Database management +wrangler d1 create +wrangler d1 list +wrangler d1 delete + +# Migrations +wrangler d1 migrations create # Create new migration file +wrangler d1 migrations apply --remote # Apply pending migrations +wrangler d1 migrations apply --local # Apply locally +wrangler d1 migrations list --remote # Show applied migrations + +# Direct SQL execution +wrangler d1 execute --remote --command="SELECT * FROM users" +wrangler d1 execute --local --file=./schema.sql + +# Backups & Import/Export +wrangler d1 export --remote --output=./backup.sql # Full export with schema +wrangler d1 export --remote --no-schema --output=./data.sql # Data only +wrangler d1 time-travel restore --timestamp="2024-01-15T14:30:00Z" # Point-in-time recovery + +# Development +wrangler dev --persist-to=./.wrangler/state +``` + +## Reading Order + +**Start here**: Quick Start above → configuration.md (setup) → api.md (queries) + +**Common tasks**: +- First time setup: configuration.md → Run migrations +- Adding queries: api.md → Prepared statements +- Pagination/caching: patterns.md +- Production optimization: Read Replication + Sessions API (this file) +- Debugging: gotchas.md + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc setup, migrations, TypeScript types, ORMs, local dev +- [api.md](./api.md) - Query methods (.all/.first/.run/.raw), batch, sessions, read replicas, error handling +- [patterns.md](./patterns.md) - Pagination, bulk operations, caching, multi-tenant, sessions, analytics +- [gotchas.md](./gotchas.md) - SQL injection, limits by plan tier, performance, common errors + +## See Also + +- [workers](../workers/) - Worker runtime and fetch handler patterns +- [hyperdrive](../hyperdrive/) - Connection pooling for external databases diff --git a/plugins/cloudflare/skills/cloudflare/references/d1/api.md b/plugins/cloudflare/skills/cloudflare/references/d1/api.md new file mode 100644 index 0000000..b3c26de --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/d1/api.md @@ -0,0 +1,196 @@ +# D1 API Reference + +## Prepared Statements (Required for Security) + +```typescript +// ❌ NEVER: Direct string interpolation (SQL injection risk) +const result = await env.DB.prepare(`SELECT * FROM users WHERE id = ${userId}`).all(); + +// ✅ CORRECT: Prepared statements with bind() +const result = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).all(); + +// Multiple parameters +const result = await env.DB.prepare('SELECT * FROM users WHERE email = ? AND active = ?').bind(email, true).all(); +``` + +## Query Execution Methods + +```typescript +// .all() - Returns all rows +const { results, success, meta } = await env.DB.prepare('SELECT * FROM users WHERE active = ?').bind(true).all(); +// results: Array of row objects; success: boolean +// meta: { duration: number, rows_read: number, rows_written: number } + +// .first() - Returns first row or null +const user = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); + +// .first(columnName) - Returns single column value +const email = await env.DB.prepare('SELECT email FROM users WHERE id = ?').bind(userId).first('email'); +// Returns string | number | null + +// .run() - For INSERT/UPDATE/DELETE (no row data returned) +const result = await env.DB.prepare('UPDATE users SET last_login = ? WHERE id = ?').bind(Date.now(), userId).run(); +// result.meta: { duration, rows_read, rows_written, last_row_id, changes } + +// .raw() - Returns array of arrays (efficient for large datasets) +const rawResults = await env.DB.prepare('SELECT id, name FROM users').raw(); +// [[1, 'Alice'], [2, 'Bob']] +``` + +## Batch Operations + +```typescript +// Execute multiple queries in single round trip (atomic transaction) +const results = await env.DB.batch([ + env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(1), + env.DB.prepare('SELECT * FROM posts WHERE author_id = ?').bind(1), + env.DB.prepare('UPDATE users SET last_access = ? WHERE id = ?').bind(Date.now(), 1) +]); +// results is array: [result1, result2, result3] + +// Batch with same prepared statement, different params +const userIds = [1, 2, 3]; +const stmt = env.DB.prepare('SELECT * FROM users WHERE id = ?'); +const results = await env.DB.batch(userIds.map(id => stmt.bind(id))); +``` + +## Transactions (via batch) + +```typescript +// D1 executes batch() as atomic transaction - all succeed or all fail +const results = await env.DB.batch([ + env.DB.prepare('INSERT INTO accounts (id, balance) VALUES (?, ?)').bind(1, 100), + env.DB.prepare('INSERT INTO accounts (id, balance) VALUES (?, ?)').bind(2, 200), + env.DB.prepare('UPDATE accounts SET balance = balance - ? WHERE id = ?').bind(50, 1), + env.DB.prepare('UPDATE accounts SET balance = balance + ? WHERE id = ?').bind(50, 2) +]); +``` + +## Sessions API (Paid Plans) + +Long-running sessions for operations exceeding 30s timeout (up to 15 min). + +```typescript +const session = env.DB.withSession({ timeout: 600 }); // 10 min (1-900s) +try { + await session.prepare('CREATE INDEX idx_large ON big_table(column)').run(); + await session.prepare('ANALYZE').run(); +} finally { + session.close(); // CRITICAL: always close to prevent leaks +} +``` + +**Use cases**: Migrations, ANALYZE, large index creation, bulk transformations + +## Read Replication (Paid Plans) + +Routes queries to nearest replica for lower latency. Writes always go to primary. + +```typescript +interface Env { + DB: D1Database; // Primary (writes) + DB_REPLICA: D1Database; // Replica (reads) +} + +// Reads: use replica +const user = await env.DB_REPLICA.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); + +// Writes: use primary +await env.DB.prepare('UPDATE users SET last_login = ? WHERE id = ?').bind(Date.now(), userId).run(); + +// Read-after-write: use primary for consistency (replication lag <100ms-2s) +await env.DB.prepare('INSERT INTO posts (title) VALUES (?)').bind(title).run(); +const post = await env.DB.prepare('SELECT * FROM posts WHERE title = ?').bind(title).first(); // Primary +``` + +## Error Handling + +```typescript +async function getUser(userId: number, env: Env): Promise { + try { + const result = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).all(); + if (!result.success) return new Response('Database error', { status: 500 }); + if (result.results.length === 0) return new Response('User not found', { status: 404 }); + return Response.json(result.results[0]); + } catch (error) { + return new Response('Internal error', { status: 500 }); + } +} + +// Constraint violations +try { + await env.DB.prepare('INSERT INTO users (email, name) VALUES (?, ?)').bind(email, name).run(); +} catch (error) { + if (error.message?.includes('UNIQUE constraint failed')) return new Response('Email exists', { status: 409 }); + throw error; +} +``` + +## REST API (HTTP) Access + +Access D1 from external services (non-Worker contexts) using Cloudflare API. + +```typescript +// Single query +const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/d1/database/${DATABASE_ID}/query`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${CLOUDFLARE_API_TOKEN}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + sql: 'SELECT * FROM users WHERE id = ?', + params: [userId] + }) + } +); + +const { result, success, errors } = await response.json(); +// result: [{ results: [...], success: true, meta: {...} }] + +// Batch queries via HTTP +const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/d1/database/${DATABASE_ID}/query`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${CLOUDFLARE_API_TOKEN}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify([ + { sql: 'SELECT * FROM users WHERE id = ?', params: [1] }, + { sql: 'SELECT * FROM posts WHERE author_id = ?', params: [1] } + ]) + } +); +``` + +**Use cases**: Server-side scripts, CI/CD migrations, administrative tools, non-Worker integrations + +## Testing & Debugging + +```typescript +// Vitest with unstable_dev +import { unstable_dev } from 'wrangler'; +describe('D1', () => { + let worker: Awaited>; + beforeAll(async () => { worker = await unstable_dev('src/index.ts'); }); + afterAll(async () => { await worker.stop(); }); + it('queries users', async () => { expect((await worker.fetch('/users')).status).toBe(200); }); +}); + +// Debug query performance +const result = await env.DB.prepare('SELECT * FROM users').all(); +console.log('Duration:', result.meta.duration, 'ms'); + +// Query plan analysis +const plan = await env.DB.prepare('EXPLAIN QUERY PLAN SELECT * FROM users WHERE email = ?').bind(email).all(); +``` + +```bash +# Inspect local database +sqlite3 .wrangler/state/v3/d1/.sqlite +.tables; .schema users; PRAGMA table_info(users); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/d1/configuration.md b/plugins/cloudflare/skills/cloudflare/references/d1/configuration.md new file mode 100644 index 0000000..8a073fc --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/d1/configuration.md @@ -0,0 +1,188 @@ +# D1 Configuration + +## wrangler.jsonc Setup + +```jsonc +{ + "name": "your-worker-name", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + "d1_databases": [ + { + "binding": "DB", // Env variable name + "database_name": "your-db-name", // Human-readable name + "database_id": "your-database-id", // UUID from dashboard/CLI + "migrations_dir": "migrations" // Optional: default is "migrations" + }, + // Read replica (paid plans only) + { + "binding": "DB_REPLICA", + "database_name": "your-db-name", + "database_id": "your-database-id" // Same ID, different binding + }, + // Multiple databases + { + "binding": "ANALYTICS_DB", + "database_name": "analytics-db", + "database_id": "yyy-yyy-yyy" + } + ] +} +``` + +## TypeScript Types + +```typescript +interface Env { DB: D1Database; ANALYTICS_DB?: D1Database; } + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const result = await env.DB.prepare('SELECT * FROM users').all(); + return Response.json(result.results); + } +} +``` + +## Migrations + +File structure: `migrations/0001_initial_schema.sql`, `0002_add_posts.sql`, etc. + +### Example Migration + +```sql +-- migrations/0001_initial_schema.sql +CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT UNIQUE NOT NULL, + name TEXT NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_users_email ON users(email); + +CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + title TEXT NOT NULL, + content TEXT, + published BOOLEAN DEFAULT 0, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE +); + +CREATE INDEX idx_posts_user_id ON posts(user_id); +CREATE INDEX idx_posts_published ON posts(published); +``` + +### Running Migrations + +```bash +# Create new migration file +wrangler d1 migrations create add_users_table +# Creates: migrations/0001_add_users_table.sql + +# Apply migrations +wrangler d1 migrations apply --local # Apply to local DB +wrangler d1 migrations apply --remote # Apply to production DB + +# List applied migrations +wrangler d1 migrations list --remote + +# Direct SQL execution (bypasses migration tracking) +wrangler d1 execute --remote --command="SELECT * FROM users" +wrangler d1 execute --local --file=./schema.sql +``` + +**Migration tracking**: Wrangler creates `d1_migrations` table automatically to track applied migrations + +## Indexing Strategy + +```sql +-- Index frequently queried columns +CREATE INDEX idx_users_email ON users(email); + +-- Composite indexes for multi-column queries +CREATE INDEX idx_posts_user_published ON posts(user_id, published); + +-- Covering indexes (include queried columns) +CREATE INDEX idx_users_email_name ON users(email, name); + +-- Partial indexes for filtered queries +CREATE INDEX idx_active_users ON users(email) WHERE active = 1; + +-- Check if query uses index +EXPLAIN QUERY PLAN SELECT * FROM users WHERE email = ?; +``` + +## Drizzle ORM + +```typescript +// drizzle.config.ts +export default { + schema: './src/schema.ts', out: './migrations', dialect: 'sqlite', driver: 'd1-http', + dbCredentials: { accountId: process.env.CLOUDFLARE_ACCOUNT_ID!, databaseId: process.env.D1_DATABASE_ID!, token: process.env.CLOUDFLARE_API_TOKEN! } +} satisfies Config; + +// schema.ts +import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core'; +export const users = sqliteTable('users', { + id: integer('id').primaryKey({ autoIncrement: true }), + email: text('email').notNull().unique(), + name: text('name').notNull() +}); + +// worker.ts +import { drizzle } from 'drizzle-orm/d1'; +import { users } from './schema'; +export default { + async fetch(request: Request, env: Env) { + const db = drizzle(env.DB); + return Response.json(await db.select().from(users)); + } +} +``` + +## Import & Export + +```bash +# Export full database (schema + data) +wrangler d1 export --remote --output=./backup.sql + +# Export data only (no schema) +wrangler d1 export --remote --no-schema --output=./data-only.sql + +# Export with foreign key constraints preserved +# (Default: foreign keys are disabled during export for import compatibility) + +# Import SQL file +wrangler d1 execute --remote --file=./backup.sql + +# Limitations +# - BLOB data may not export correctly (use R2 for binary files) +# - Very large exports (>1GB) may timeout (split into chunks) +# - Import is NOT atomic (use batch() for transactional imports in Workers) +``` + +## Plan Tiers + +| Feature | Free | Paid | +|---------|------|------| +| Database size | 500 MB | 10 GB | +| Batch size | 1,000 statements | 10,000 statements | +| Time Travel | 7 days | 30 days | +| Read replicas | ❌ | ✅ | +| Sessions API | ❌ | ✅ (up to 15 min) | +| Pricing | Free | $5/mo + usage | + +**Usage pricing** (paid plans): $0.001 per 1K reads + $1 per 1M writes + $0.75/GB storage/month + +## Local Development + +```bash +wrangler dev --persist-to=./.wrangler/state # Persist across restarts +# Local DB: .wrangler/state/v3/d1/.sqlite +sqlite3 .wrangler/state/v3/d1/.sqlite # Inspect + +# Local dev uses free tier limits by default +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/d1/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/d1/gotchas.md new file mode 100644 index 0000000..9f9a95a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/d1/gotchas.md @@ -0,0 +1,98 @@ +# D1 Gotchas & Troubleshooting + +## Common Errors + +### "SQL Injection Vulnerability" + +**Cause:** Using string interpolation instead of prepared statements with bind() +**Solution:** ALWAYS use prepared statements: `env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).all()` instead of string interpolation which allows attackers to inject malicious SQL + +### "no such table" + +**Cause:** Table doesn't exist because migrations haven't been run, or using wrong database binding +**Solution:** Run migrations using `wrangler d1 migrations apply --remote` and verify binding name in wrangler.jsonc matches code + +### "UNIQUE constraint failed" + +**Cause:** Attempting to insert duplicate value in column with UNIQUE constraint +**Solution:** Catch error and return 409 Conflict status code + +### "Query Timeout (30s exceeded)" + +**Cause:** Query execution exceeds 30 second timeout limit +**Solution:** Break into smaller queries, add indexes to speed up queries, or reduce dataset size + +### "N+1 Query Problem" + +**Cause:** Making multiple individual queries in a loop instead of single optimized query +**Solution:** Use JOIN to fetch related data in single query or use `batch()` method for multiple queries + +### "Missing Indexes" + +**Cause:** Queries performing full table scans without indexes +**Solution:** Use `EXPLAIN QUERY PLAN` to check if index is used, then create index with `CREATE INDEX idx_users_email ON users(email)` + +### "Boolean Type Issues" + +**Cause:** SQLite uses INTEGER (0/1) not native boolean type +**Solution:** Bind 1 or 0 instead of true/false when working with boolean values + +### "Date/Time Type Issues" + +**Cause:** SQLite doesn't have native DATE/TIME types +**Solution:** Use TEXT (ISO 8601 format) or INTEGER (unix timestamp) for date/time values + +## Plan Tier Limits + +| Limit | Free Tier | Paid Plans | Notes | +|-------|-----------|------------|-------| +| Database size | 500 MB | 10 GB | Design for multiple DBs per tenant on paid | +| Row size | 1 MB | 1 MB | Store large files in R2, not D1 | +| Query timeout | 30s | 30s (900s with sessions) | Use sessions API for migrations | +| Batch size | 1,000 statements | 10,000 statements | Split large batches accordingly | +| Time Travel | 7 days | 30 days | Point-in-time recovery window | +| Read replicas | ❌ Not available | ✅ Available | Paid add-on for lower latency | +| Sessions API | ❌ Not available | ✅ Up to 15 min | For migrations and heavy operations | +| Concurrent requests | 10,000/min | Higher | Contact support for custom limits | + +## Production Gotchas + +### "Batch size exceeded" + +**Cause:** Attempting to send >1,000 statements on free tier or >10,000 on paid +**Solution:** Chunk batches: `for (let i = 0; i < stmts.length; i += MAX_BATCH) await env.DB.batch(stmts.slice(i, i + MAX_BATCH))` + +### "Session not closed / resource leak" + +**Cause:** Forgot to call `session.close()` after using sessions API +**Solution:** Always use try/finally block: `try { await session.prepare(...) } finally { session.close() }` + +### "Replication lag causing stale reads" + +**Cause:** Reading from replica immediately after write - replication lag can be 100ms-2s +**Solution:** Use primary for read-after-write: `await env.DB.prepare(...)` not `env.DB_REPLICA` + +### "Migration applied to local but not remote" + +**Cause:** Forgot `--remote` flag when applying migrations +**Solution:** Always run `wrangler d1 migrations apply --remote` for production + +### "Foreign key constraint failed" + +**Cause:** Inserting row with FK to non-existent parent, or deleting parent before children +**Solution:** Enable FK enforcement: `PRAGMA foreign_keys = ON;` and use ON DELETE CASCADE in schema + +### "BLOB data corrupted on export" + +**Cause:** D1 export may not handle BLOB correctly +**Solution:** Store binary files in R2, only store R2 URLs/keys in D1 + +### "Database size approaching limit" + +**Cause:** Storing too much data in single database +**Solution:** Horizontal scale-out: create per-tenant/per-user databases, archive old data, or upgrade to paid plan + +### "Local dev vs production behavior differs" + +**Cause:** Local uses SQLite file, production uses distributed D1 - different performance/limits +**Solution:** Always test migrations on remote with `--remote` flag before production rollout diff --git a/plugins/cloudflare/skills/cloudflare/references/d1/patterns.md b/plugins/cloudflare/skills/cloudflare/references/d1/patterns.md new file mode 100644 index 0000000..f01c7bd --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/d1/patterns.md @@ -0,0 +1,189 @@ +# D1 Patterns & Best Practices + +## Pagination + +```typescript +async function getUsers({ page, pageSize }: { page: number; pageSize: number }, env: Env) { + const offset = (page - 1) * pageSize; + const [countResult, dataResult] = await env.DB.batch([ + env.DB.prepare('SELECT COUNT(*) as total FROM users'), + env.DB.prepare('SELECT * FROM users ORDER BY created_at DESC LIMIT ? OFFSET ?').bind(pageSize, offset) + ]); + return { data: dataResult.results, total: countResult.results[0].total, page, pageSize, totalPages: Math.ceil(countResult.results[0].total / pageSize) }; +} +``` + +## Conditional Queries + +```typescript +async function searchUsers(filters: { name?: string; email?: string; active?: boolean }, env: Env) { + const conditions: string[] = [], params: (string | number | boolean | null)[] = []; + if (filters.name) { conditions.push('name LIKE ?'); params.push(`%${filters.name}%`); } + if (filters.email) { conditions.push('email = ?'); params.push(filters.email); } + if (filters.active !== undefined) { conditions.push('active = ?'); params.push(filters.active ? 1 : 0); } + const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + return await env.DB.prepare(`SELECT * FROM users ${whereClause}`).bind(...params).all(); +} +``` + +## Bulk Insert + +```typescript +async function bulkInsertUsers(users: Array<{ name: string; email: string }>, env: Env) { + const stmt = env.DB.prepare('INSERT INTO users (name, email) VALUES (?, ?)'); + const batch = users.map(user => stmt.bind(user.name, user.email)); + return await env.DB.batch(batch); +} +``` + +## Caching with KV + +```typescript +async function getCachedUser(userId: number, env: { DB: D1Database; CACHE: KVNamespace }) { + const cacheKey = `user:${userId}`; + const cached = await env.CACHE?.get(cacheKey, 'json'); + if (cached) return cached; + const user = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); + if (user) await env.CACHE?.put(cacheKey, JSON.stringify(user), { expirationTtl: 300 }); + return user; +} +``` + +## Query Optimization + +```typescript +// ✅ Use indexes in WHERE clauses +const users = await env.DB.prepare('SELECT * FROM users WHERE email = ?').bind(email).all(); + +// ✅ Limit result sets +const recentPosts = await env.DB.prepare('SELECT * FROM posts ORDER BY created_at DESC LIMIT 100').all(); + +// ✅ Use batch() for multiple independent queries +const [user, posts, comments] = await env.DB.batch([ + env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId), + env.DB.prepare('SELECT * FROM posts WHERE user_id = ?').bind(userId), + env.DB.prepare('SELECT * FROM comments WHERE user_id = ?').bind(userId) +]); + +// ❌ Avoid N+1 queries +for (const post of posts) { + const author = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(post.user_id).first(); // Bad: multiple round trips +} + +// ✅ Use JOINs instead +const postsWithAuthors = await env.DB.prepare(` + SELECT posts.*, users.name as author_name + FROM posts + JOIN users ON posts.user_id = users.id +`).all(); +``` + +## Multi-Tenant SaaS + +```typescript +// Each tenant gets own database +export default { + async fetch(request: Request, env: { [key: `TENANT_${string}`]: D1Database }) { + const tenantId = request.headers.get('X-Tenant-ID'); + const data = await env[`TENANT_${tenantId}`].prepare('SELECT * FROM records').all(); + return Response.json(data.results); + } +} +``` + +## Session Storage + +```typescript +async function createSession(userId: number, token: string, env: Env) { + const expiresAt = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString(); + return await env.DB.prepare('INSERT INTO sessions (user_id, token, expires_at) VALUES (?, ?, ?)').bind(userId, token, expiresAt).run(); +} + +async function validateSession(token: string, env: Env) { + return await env.DB.prepare('SELECT s.*, u.email FROM sessions s JOIN users u ON s.user_id = u.id WHERE s.token = ? AND s.expires_at > CURRENT_TIMESTAMP').bind(token).first(); +} +``` + +## Analytics/Events + +```typescript +async function logEvent(event: { type: string; userId?: number; metadata: object }, env: Env) { + return await env.DB.prepare('INSERT INTO events (type, user_id, metadata) VALUES (?, ?, ?)').bind(event.type, event.userId || null, JSON.stringify(event.metadata)).run(); +} + +async function getEventStats(startDate: string, endDate: string, env: Env) { + return await env.DB.prepare('SELECT type, COUNT(*) as count FROM events WHERE timestamp BETWEEN ? AND ? GROUP BY type ORDER BY count DESC').bind(startDate, endDate).all(); +} +``` + +## Read Replication Pattern (Paid Plans) + +```typescript +interface Env { DB: D1Database; DB_REPLICA: D1Database; } + +export default { + async fetch(request: Request, env: Env) { + if (request.method === 'GET') { + // Reads: use replica for lower latency + const users = await env.DB_REPLICA.prepare('SELECT * FROM users WHERE active = 1').all(); + return Response.json(users.results); + } + + if (request.method === 'POST') { + const { name, email } = await request.json(); + const result = await env.DB.prepare('INSERT INTO users (name, email) VALUES (?, ?)').bind(name, email).run(); + + // Read-after-write: use primary for consistency (replication lag <100ms-2s) + const user = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(result.meta.last_row_id).first(); + return Response.json(user, { status: 201 }); + } + } +} +``` + +**Use replicas for**: Analytics dashboards, search results, public queries (eventual consistency OK) +**Use primary for**: Read-after-write, financial transactions, authentication (consistency required) + +## Sessions API Pattern (Paid Plans) + +```typescript +// Migration with long-running session (up to 15 min) +async function runMigration(env: Env) { + const session = env.DB.withSession({ timeout: 600 }); // 10 min + try { + await session.prepare('CREATE INDEX idx_users_email ON users(email)').run(); + await session.prepare('CREATE INDEX idx_posts_user ON posts(user_id)').run(); + await session.prepare('ANALYZE').run(); + } finally { + session.close(); // Always close to prevent leaks + } +} + +// Bulk transformation with batching +async function transformLargeDataset(env: Env) { + const session = env.DB.withSession({ timeout: 900 }); // 15 min max + try { + const BATCH_SIZE = 1000; + let offset = 0; + while (true) { + const rows = await session.prepare('SELECT id, data FROM legacy LIMIT ? OFFSET ?').bind(BATCH_SIZE, offset).all(); + if (rows.results.length === 0) break; + const updates = rows.results.map(row => + session.prepare('UPDATE legacy SET new_data = ? WHERE id = ?').bind(transform(row.data), row.id) + ); + await session.batch(updates); + offset += BATCH_SIZE; + } + } finally { session.close(); } +} +``` + +## Time Travel & Backups + +```bash +wrangler d1 time-travel restore --timestamp="2024-01-15T14:30:00Z" # Point-in-time +wrangler d1 time-travel info # List restore points (7 days free, 30 days paid) +wrangler d1 export --remote --output=./backup.sql # Full export +wrangler d1 export --remote --no-schema --output=./data.sql # Data only +wrangler d1 execute --remote --file=./backup.sql # Import +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/ddos/README.md b/plugins/cloudflare/skills/cloudflare/references/ddos/README.md new file mode 100644 index 0000000..117dd21 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ddos/README.md @@ -0,0 +1,41 @@ +# Cloudflare DDoS Protection + +Autonomous, always-on protection against DDoS attacks across L3/4 and L7. + +## Protection Types + +- **HTTP DDoS (L7)**: Protects HTTP/HTTPS traffic, phase `ddos_l7`, zone/account level +- **Network DDoS (L3/4)**: UDP/SYN/DNS floods, phase `ddos_l4`, account level only +- **Adaptive DDoS**: Learns 7-day baseline, detects deviations, 4 profile types (Origins, User-Agents, Locations, Protocols) + +## Plan Availability + +| Feature | Free | Pro | Business | Enterprise | Enterprise Advanced | +|---------|------|-----|----------|------------|---------------------| +| HTTP DDoS (L7) | ✓ | ✓ | ✓ | ✓ | ✓ | +| Network DDoS (L3/4) | ✓ | ✓ | ✓ | ✓ | ✓ | +| Override rules | 1 | 1 | 1 | 1 | 10 | +| Custom expressions | ✗ | ✗ | ✗ | ✗ | ✓ | +| Log action | ✗ | ✗ | ✗ | ✗ | ✓ | +| Adaptive DDoS | ✗ | ✗ | ✗ | ✓ | ✓ | +| Alert filters | Basic | Basic | Basic | Advanced | Advanced | + +## Actions & Sensitivity + +- **Actions**: `block`, `managed_challenge`, `challenge`, `log` (Enterprise Advanced only) +- **Sensitivity**: `default` (high), `medium`, `low`, `eoff` (essentially off) +- **Override**: By category/tag or individual rule ID +- **Scope**: Zone-level overrides take precedence over account-level + +## Reading Order + +| File | Purpose | Start Here If... | +|------|---------|------------------| +| [configuration.md](./configuration.md) | Dashboard setup, rule structure, adaptive profiles | You're setting up DDoS protection for the first time | +| [api.md](./api.md) | API endpoints, SDK usage, ruleset ID discovery | You're automating configuration or need programmatic access | +| [patterns.md](./patterns.md) | Protection strategies, defense-in-depth, dynamic response | You need implementation patterns or layered security | +| [gotchas.md](./gotchas.md) | False positives, tuning, error handling | You're troubleshooting or optimizing existing protection | + +## See Also +- [waf](../waf/) - Application-layer security rules +- [bot-management](../bot-management/) - Bot detection and mitigation diff --git a/plugins/cloudflare/skills/cloudflare/references/ddos/api.md b/plugins/cloudflare/skills/cloudflare/references/ddos/api.md new file mode 100644 index 0000000..b96284a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ddos/api.md @@ -0,0 +1,164 @@ +# DDoS API + +## Endpoints + +### HTTP DDoS (L7) + +```typescript +// Zone-level +PUT /zones/{zoneId}/rulesets/phases/ddos_l7/entrypoint +GET /zones/{zoneId}/rulesets/phases/ddos_l7/entrypoint + +// Account-level (Enterprise Advanced) +PUT /accounts/{accountId}/rulesets/phases/ddos_l7/entrypoint +GET /accounts/{accountId}/rulesets/phases/ddos_l7/entrypoint +``` + +### Network DDoS (L3/4) + +```typescript +// Account-level only +PUT /accounts/{accountId}/rulesets/phases/ddos_l4/entrypoint +GET /accounts/{accountId}/rulesets/phases/ddos_l4/entrypoint +``` + +## TypeScript SDK + +**SDK Version**: Requires `cloudflare` >= 3.0.0 for ruleset phase methods. + +```typescript +import Cloudflare from "cloudflare"; + +const client = new Cloudflare({ apiToken: process.env.CLOUDFLARE_API_TOKEN }); + +// STEP 1: Discover managed ruleset ID (required for overrides) +const allRulesets = await client.rulesets.list({ zone_id: zoneId }); +const ddosRuleset = allRulesets.result.find( + (r) => r.kind === "managed" && r.phase === "ddos_l7" +); +if (!ddosRuleset) throw new Error("DDoS managed ruleset not found"); +const managedRulesetId = ddosRuleset.id; + +// STEP 2: Get current HTTP DDoS configuration +const entrypointRuleset = await client.zones.rulesets.phases.entrypoint.get("ddos_l7", { + zone_id: zoneId, +}); + +// STEP 3: Update HTTP DDoS ruleset with overrides +await client.zones.rulesets.phases.entrypoint.update("ddos_l7", { + zone_id: zoneId, + rules: [ + { + action: "execute", + expression: "true", + action_parameters: { + id: managedRulesetId, // From discovery step + overrides: { + sensitivity_level: "medium", + action: "managed_challenge", + }, + }, + }, + ], +}); + +// Network DDoS (account level, L3/4) +const l4Rulesets = await client.rulesets.list({ account_id: accountId }); +const l4DdosRuleset = l4Rulesets.result.find( + (r) => r.kind === "managed" && r.phase === "ddos_l4" +); +const l4Ruleset = await client.accounts.rulesets.phases.entrypoint.get("ddos_l4", { + account_id: accountId, +}); +``` + +## Alert Configuration + +```typescript +interface DDoSAlertConfig { + name: string; + enabled: boolean; + alert_type: "http_ddos_attack_alert" | "layer_3_4_ddos_attack_alert" + | "advanced_http_ddos_attack_alert" | "advanced_layer_3_4_ddos_attack_alert"; + filters?: { + zones?: string[]; + hostnames?: string[]; + requests_per_second?: number; + packets_per_second?: number; + megabits_per_second?: number; + ip_prefixes?: string[]; // CIDR + ip_addresses?: string[]; + protocols?: string[]; + }; + mechanisms: { + email?: Array<{ id: string }>; + webhooks?: Array<{ id: string }>; + pagerduty?: Array<{ id: string }>; + }; +} + +// Create alert +await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/alerting/v3/policies`, + { + method: "POST", + headers: { + Authorization: `Bearer ${apiToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(alertConfig), + } +); +``` + +## Typed Override Examples + +```typescript +// Override by category +interface CategoryOverride { + action: "execute"; + expression: string; + action_parameters: { + id: string; + overrides: { + categories?: Array<{ + category: "http-flood" | "http-anomaly" | "udp-flood" | "syn-flood"; + sensitivity_level?: "default" | "medium" | "low" | "eoff"; + action?: "block" | "managed_challenge" | "challenge" | "log"; + }>; + }; + }; +} + +// Override by rule ID +interface RuleOverride { + action: "execute"; + expression: string; + action_parameters: { + id: string; + overrides: { + rules?: Array<{ + id: string; + action?: "block" | "managed_challenge" | "challenge" | "log"; + sensitivity_level?: "default" | "medium" | "low" | "eoff"; + }>; + }; + }; +} + +// Example: Override specific adaptive rule +const adaptiveOverride: RuleOverride = { + action: "execute", + expression: "true", + action_parameters: { + id: managedRulesetId, + overrides: { + rules: [ + { id: "...adaptive-origins-rule-id...", sensitivity_level: "low" }, + ], + }, + }, +}; +``` + +See [patterns.md](./patterns.md) for complete implementation patterns. diff --git a/plugins/cloudflare/skills/cloudflare/references/ddos/configuration.md b/plugins/cloudflare/skills/cloudflare/references/ddos/configuration.md new file mode 100644 index 0000000..14c6e32 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ddos/configuration.md @@ -0,0 +1,93 @@ +# DDoS Configuration + +## Dashboard Setup + +1. Navigate to Security > DDoS +2. Select HTTP DDoS or Network-layer DDoS +3. Configure sensitivity & action per ruleset/category/rule +4. Apply overrides with optional expressions (Enterprise Advanced) +5. Enable Adaptive DDoS toggle (Enterprise/Enterprise Advanced, requires 7 days traffic history) + +## Rule Structure + +```typescript +interface DDoSOverride { + description: string; + rules: Array<{ + action: "execute"; + expression: string; // Custom expression (Enterprise Advanced) or "true" for all + action_parameters: { + id: string; // Managed ruleset ID (discover via api.md) + overrides: { + sensitivity_level?: "default" | "medium" | "low" | "eoff"; + action?: "block" | "managed_challenge" | "challenge" | "log"; // log = Enterprise Advanced only + categories?: Array<{ + category: string; // e.g., "http-flood", "udp-flood" + sensitivity_level?: string; + }>; + rules?: Array<{ + id: string; + action?: string; + sensitivity_level?: string; + }>; + }; + }; + }>; +} +``` + +## Expression Availability + +| Plan | Custom Expressions | Example | +|------|-------------------|---------| +| Free/Pro/Business | ✗ | Use `"true"` only | +| Enterprise | ✗ | Use `"true"` only | +| Enterprise Advanced | ✓ | `ip.src in {...}`, `http.request.uri.path matches "..."` | + +## Sensitivity Mapping + +| UI | API | Threshold | +|----|-----|-----------| +| High | `default` | Most aggressive | +| Medium | `medium` | Balanced | +| Low | `low` | Less aggressive | +| Essentially Off | `eoff` | Minimal mitigation | + +## Common Categories + +- `http-flood`, `http-anomaly` (L7) +- `udp-flood`, `syn-flood`, `dns-flood` (L3/4) + +## Override Precedence + +Multiple override layers apply in this order (higher precedence wins): + +``` +Zone-level > Account-level +Individual Rule > Category > Global sensitivity/action +``` + +**Example**: Zone rule for `/api/*` overrides account-level global settings. + +## Adaptive DDoS Profiles + +**Availability**: Enterprise, Enterprise Advanced +**Learning period**: 7 days of traffic history required + +| Profile Type | Description | Detects | +|--------------|-------------|---------| +| **Origins** | Traffic patterns per origin server | Anomalous requests to specific origins | +| **User-Agents** | Traffic patterns per User-Agent | Malicious/anomalous user agent strings | +| **Locations** | Traffic patterns per geo-location | Attacks from specific countries/regions | +| **Protocols** | Traffic patterns per protocol (L3/4) | Protocol-specific flood attacks | + +Configure by targeting specific adaptive rule IDs via API (see api.md#typed-override-examples). + +## Alerting + +Configure via Notifications: +- Alert types: `http_ddos_attack_alert`, `layer_3_4_ddos_attack_alert`, `advanced_*` variants +- Filters: zones, hostnames, RPS/PPS/Mbps thresholds, IPs, protocols +- Mechanisms: email, webhooks, PagerDuty + +See [api.md](./api.md#alert-configuration) for API examples. diff --git a/plugins/cloudflare/skills/cloudflare/references/ddos/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/ddos/gotchas.md new file mode 100644 index 0000000..f2a97d1 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ddos/gotchas.md @@ -0,0 +1,107 @@ +# DDoS Gotchas + +## Common Errors + +### "False positives blocking legitimate traffic" + +**Cause**: Sensitivity too high, wrong action, or missing exceptions +**Solution**: +1. Lower sensitivity for specific rule/category +2. Use `log` action first to validate (Enterprise Advanced) +3. Add exception with custom expression (e.g., allowlist IPs) +4. Query flagged requests via GraphQL Analytics API to identify patterns + +### "Attacks getting through" + +**Cause**: Sensitivity too low or wrong action +**Solution**: Increase to `default` sensitivity and use `block` action: +```typescript +const config = { + rules: [{ + expression: "true", + action: "execute", + action_parameters: { id: managedRulesetId, overrides: { sensitivity_level: "default", action: "block" } }, + }], +}; +``` + +### "Adaptive rules not working" + +**Cause**: Insufficient traffic history (needs 7 days) +**Solution**: Wait for baseline to establish, check dashboard for adaptive rule status + +### "Zone override ignored" + +**Cause**: Account overrides conflict with zone overrides +**Solution**: Configure at zone level OR remove zone overrides to use account-level + +### "Log action not available" + +**Cause**: Not on Enterprise Advanced DDoS plan +**Solution**: Use `managed_challenge` with low sensitivity for testing + +### "Rule limit exceeded" + +**Cause**: Too many override rules (Free/Pro/Business: 1, Enterprise Advanced: 10) +**Solution**: Combine conditions in single expression using `and`/`or` + +### "Cannot override rule" + +**Cause**: Rule is read-only +**Solution**: Check API response for read-only indicator, use different rule + +### "Cannot disable DDoS protection" + +**Cause**: DDoS managed rulesets cannot be fully disabled (always-on protection) +**Solution**: Set `sensitivity_level: "eoff"` for minimal mitigation + +### "Expression not allowed" + +**Cause**: Custom expressions require Enterprise Advanced plan +**Solution**: Use `expression: "true"` for all traffic, or upgrade plan + +### "Managed ruleset not found" + +**Cause**: Zone/account doesn't have DDoS managed ruleset, or incorrect phase +**Solution**: Verify ruleset exists via `client.rulesets.list()`, check phase name (`ddos_l7` or `ddos_l4`) + +## API Error Codes + +| Error Code | Message | Cause | Solution | +|------------|---------|-------|----------| +| 10000 | Authentication error | Invalid/missing API token | Check token has DDoS permissions | +| 81000 | Ruleset validation failed | Invalid rule structure | Verify `action_parameters.id` is managed ruleset ID | +| 81020 | Expression not allowed | Custom expressions on wrong plan | Use `"true"` or upgrade to Enterprise Advanced | +| 81021 | Rule limit exceeded | Too many override rules | Reduce rules or upgrade (Enterprise Advanced: 10) | +| 81022 | Invalid sensitivity level | Wrong sensitivity value | Use: `default`, `medium`, `low`, `eoff` | +| 81023 | Invalid action | Wrong action for plan | Enterprise Advanced only: `log` action | + +## Limits + +| Resource/Limit | Free/Pro/Business | Enterprise | Enterprise Advanced | +|----------------|-------------------|------------|---------------------| +| Override rules per zone | 1 | 1 | 10 | +| Custom expressions | ✗ | ✗ | ✓ | +| Log action | ✗ | ✗ | ✓ | +| Adaptive DDoS | ✗ | ✓ | ✓ | +| Traffic history required | - | 7 days | 7 days | + +## Tuning Strategy + +1. Start with `log` action + `medium` sensitivity +2. Monitor for 24-48 hours +3. Identify false positives, add exceptions +4. Gradually increase to `default` sensitivity +5. Change action from `log` → `managed_challenge` → `block` +6. Document all adjustments + +## Best Practices + +- Test during low-traffic periods +- Use zone-level for per-site tuning +- Reference IP lists for easier management +- Set appropriate alert thresholds (avoid noise) +- Combine with WAF for layered defense +- Avoid over-tuning (keep config simple) + +See [patterns.md](./patterns.md) for progressive rollout examples. diff --git a/plugins/cloudflare/skills/cloudflare/references/ddos/patterns.md b/plugins/cloudflare/skills/cloudflare/references/ddos/patterns.md new file mode 100644 index 0000000..a46ef2f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/ddos/patterns.md @@ -0,0 +1,174 @@ +# DDoS Protection Patterns + +## Allowlist Trusted IPs + +```typescript +const config = { + description: "Allowlist trusted IPs", + rules: [{ + expression: "ip.src in { 203.0.113.0/24 192.0.2.1 }", + action: "execute", + action_parameters: { + id: managedRulesetId, + overrides: { sensitivity_level: "eoff" }, + }, + }], +}; + +await client.accounts.rulesets.phases.entrypoint.update("ddos_l7", { + account_id: accountId, + ...config, +}); +``` + +## Route-specific Sensitivity + +```typescript +const config = { + description: "Route-specific protection", + rules: [ + { + expression: "not http.request.uri.path matches \"^/api/\"", + action: "execute", + action_parameters: { + id: managedRulesetId, + overrides: { sensitivity_level: "default", action: "block" }, + }, + }, + { + expression: "http.request.uri.path matches \"^/api/\"", + action: "execute", + action_parameters: { + id: managedRulesetId, + overrides: { sensitivity_level: "low", action: "managed_challenge" }, + }, + }, + ], +}; +``` + +## Progressive Enhancement + +```typescript +enum ProtectionLevel { MONITORING = "monitoring", LOW = "low", MEDIUM = "medium", HIGH = "high" } + +const levelConfig = { + [ProtectionLevel.MONITORING]: { action: "log", sensitivity: "eoff" }, + [ProtectionLevel.LOW]: { action: "managed_challenge", sensitivity: "low" }, + [ProtectionLevel.MEDIUM]: { action: "managed_challenge", sensitivity: "medium" }, + [ProtectionLevel.HIGH]: { action: "block", sensitivity: "default" }, +} as const; + +async function setProtectionLevel(zoneId: string, level: ProtectionLevel, rulesetId: string, client: Cloudflare) { + const settings = levelConfig[level]; + return client.zones.rulesets.phases.entrypoint.update("ddos_l7", { + zone_id: zoneId, + rules: [{ + expression: "true", + action: "execute", + action_parameters: { id: rulesetId, overrides: { action: settings.action, sensitivity_level: settings.sensitivity } }, + }], + }); +} +``` + +## Dynamic Response to Attacks + +```typescript +interface Env { CLOUDFLARE_API_TOKEN: string; ZONE_ID: string; KV: KVNamespace; } + +export default { + async fetch(request: Request, env: Env): Promise { + if (request.url.includes("/attack-detected")) { + const attackData = await request.json(); + await env.KV.put(`attack:${Date.now()}`, JSON.stringify(attackData), { expirationTtl: 86400 }); + const recentAttacks = await getRecentAttacks(env.KV); + if (recentAttacks.length > 5) { + await setProtectionLevel(env.ZONE_ID, ProtectionLevel.HIGH, managedRulesetId, client); + return new Response("Protection increased"); + } + } + return new Response("OK"); + }, + async scheduled(event: ScheduledEvent, env: Env): Promise { + const recentAttacks = await getRecentAttacks(env.KV); + if (recentAttacks.length === 0) await setProtectionLevel(env.ZONE_ID, ProtectionLevel.MEDIUM, managedRulesetId, client); + }, +}; +``` + +## Multi-rule Tiered Protection (Enterprise Advanced) + +```typescript +const config = { + description: "Multi-tier DDoS protection", + rules: [ + { + expression: "not ip.src in $known_ips and not cf.bot_management.score gt 30", + action: "execute", + action_parameters: { id: managedRulesetId, overrides: { sensitivity_level: "default", action: "block" } }, + }, + { + expression: "cf.bot_management.verified_bot", + action: "execute", + action_parameters: { id: managedRulesetId, overrides: { sensitivity_level: "medium", action: "managed_challenge" } }, + }, + { + expression: "ip.src in $trusted_ips", + action: "execute", + action_parameters: { id: managedRulesetId, overrides: { sensitivity_level: "low" } }, + }, + ], +}; +``` + +## Defense in Depth + +Layered security stack: DDoS + WAF + Rate Limiting + Bot Management. + +```typescript +// Layer 1: DDoS (volumetric attacks) +await client.zones.rulesets.phases.entrypoint.update("ddos_l7", { + zone_id: zoneId, + rules: [{ expression: "true", action: "execute", action_parameters: { id: ddosRulesetId, overrides: { sensitivity_level: "medium" } } }], +}); + +// Layer 2: WAF (exploit protection) +await client.zones.rulesets.phases.entrypoint.update("http_request_firewall_managed", { + zone_id: zoneId, + rules: [{ expression: "true", action: "execute", action_parameters: { id: wafRulesetId } }], +}); + +// Layer 3: Rate Limiting (abuse prevention) +await client.zones.rulesets.phases.entrypoint.update("http_ratelimit", { + zone_id: zoneId, + rules: [{ expression: "http.request.uri.path eq \"/api/login\"", action: "block", ratelimit: { characteristics: ["ip.src"], period: 60, requests_per_period: 5 } }], +}); + +// Layer 4: Bot Management (automation detection) +await client.zones.rulesets.phases.entrypoint.update("http_request_sbfm", { + zone_id: zoneId, + rules: [{ expression: "cf.bot_management.score lt 30", action: "managed_challenge" }], +}); +``` + +## Cache Strategy for DDoS Mitigation + +Exclude query strings from cache key to counter randomized query parameter attacks. + +```typescript +const cacheRule = { + expression: "http.request.uri.path matches \"^/api/\"", + action: "set_cache_settings", + action_parameters: { + cache: true, + cache_key: { ignore_query_strings_order: true, custom_key: { query_string: { exclude: { all: true } } } }, + }, +}; + +await client.zones.rulesets.phases.entrypoint.update("http_request_cache_settings", { zone_id: zoneId, rules: [cacheRule] }); +``` + +**Rationale**: Attackers randomize query strings (`?random=123456`) to bypass cache. Excluding query params ensures cache hits absorb attack traffic. + +See [configuration.md](./configuration.md) for rule structure details. diff --git a/plugins/cloudflare/skills/cloudflare/references/do-storage/README.md b/plugins/cloudflare/skills/cloudflare/references/do-storage/README.md new file mode 100644 index 0000000..426d2c4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/do-storage/README.md @@ -0,0 +1,75 @@ +# Cloudflare Durable Objects Storage + +Persistent storage API for Durable Objects with SQLite and KV backends, PITR, and automatic concurrency control. + +## Overview + +DO Storage provides: +- SQLite-backed (recommended) or KV-backed +- SQL API + synchronous/async KV APIs +- Automatic input/output gates (race-free) +- 30-day point-in-time recovery (PITR) +- Transactions and alarms + +**Use cases:** Stateful coordination, real-time collaboration, counters, sessions, rate limiters + +**Billing:** Charged by request, GB-month storage, and rowsRead/rowsWritten for SQL operations + +## Quick Start + +```typescript +export class Counter extends DurableObject { + sql: SqlStorage; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.sql = ctx.storage.sql; + this.sql.exec('CREATE TABLE IF NOT EXISTS data(key TEXT PRIMARY KEY, value INTEGER)'); + } + + async increment(): Promise { + const result = this.sql.exec( + 'INSERT INTO data VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = value + 1 RETURNING value', + 'counter', 1 + ).one(); + return result?.value || 1; + } +} +``` + +## Storage Backends + +| Backend | Create Method | APIs | PITR | +|---------|---------------|------|------| +| SQLite (recommended) | `new_sqlite_classes` | SQL + sync KV + async KV | ✅ | +| KV (legacy) | `new_classes` | async KV only | ❌ | + +## Core APIs + +- **SQL API** (`ctx.storage.sql`): Full SQLite with extensions (FTS5, JSON, math) +- **Sync KV** (`ctx.storage.kv`): Synchronous key-value (SQLite only) +- **Async KV** (`ctx.storage`): Asynchronous key-value (both backends) +- **Transactions** (`transactionSync()`, `transaction()`) +- **PITR** (`getBookmarkForTime()`, `onNextSessionRestoreBookmark()`) +- **Alarms** (`setAlarm()`, `alarm()` handler) + +## Reading Order + +**New to DO storage:** configuration.md → api.md → patterns.md → gotchas.md +**Building features:** patterns.md → api.md → gotchas.md +**Debugging issues:** gotchas.md → api.md +**Writing tests:** testing.md + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc migrations, SQLite vs KV setup, RPC binding +- [api.md](./api.md) - SQL exec/cursors, KV methods, storage options, transactions, alarms, PITR +- [patterns.md](./patterns.md) - Schema migrations, caching, rate limiting, batch processing, parent-child coordination +- [gotchas.md](./gotchas.md) - Concurrency gates, INTEGER precision, transaction rules, SQL limits +- [testing.md](./testing.md) - vitest-pool-workers setup, testing DOs with SQL/alarms/PITR + +## See Also + +- [durable-objects](../durable-objects/) - DO fundamentals and coordination patterns +- [workers](../workers/) - Worker runtime for DO stubs +- [d1](../d1/) - Shared database alternative to per-DO storage diff --git a/plugins/cloudflare/skills/cloudflare/references/do-storage/api.md b/plugins/cloudflare/skills/cloudflare/references/do-storage/api.md new file mode 100644 index 0000000..e659598 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/do-storage/api.md @@ -0,0 +1,102 @@ +# DO Storage API Reference + +## SQL API + +```typescript +const cursor = this.sql.exec('SELECT * FROM users WHERE email = ?', email); +for (let row of cursor) {} // Objects: { id, name, email } +cursor.toArray(); cursor.one(); // Single row (throws if != 1) +for (let row of cursor.raw()) {} // Arrays: [1, "Alice", "..."] + +// Manual iteration +const iter = cursor[Symbol.iterator](); +const first = iter.next(); // { value: {...}, done: false } + +cursor.columnNames; // ["id", "name", "email"] +cursor.rowsRead; cursor.rowsWritten; // Billing + +type User = { id: number; name: string; email: string }; +const user = this.sql.exec('...', userId).one(); +``` + +## Sync KV API (SQLite only) + +```typescript +this.ctx.storage.kv.get("counter"); // undefined if missing +this.ctx.storage.kv.put("counter", 42); +this.ctx.storage.kv.put("user", { name: "Alice", age: 30 }); +this.ctx.storage.kv.delete("counter"); // true if existed + +for (let [key, value] of this.ctx.storage.kv.list()) {} + +// List options: start, prefix, reverse, limit +this.ctx.storage.kv.list({ start: "user:", prefix: "user:", reverse: true, limit: 100 }); +``` + +## Async KV API (Both backends) + +```typescript +await this.ctx.storage.get("key"); // Single +await this.ctx.storage.get(["key1", "key2"]); // Multiple (max 128) +await this.ctx.storage.put("key", value); // Single +await this.ctx.storage.put({ "key1": "v1", "key2": { nested: true } }); // Multiple (max 128) +await this.ctx.storage.delete("key"); +await this.ctx.storage.delete(["key1", "key2"]); +await this.ctx.storage.list({ prefix: "user:", limit: 100 }); + +// Options: allowConcurrency, noCache, allowUnconfirmed +await this.ctx.storage.get("key", { allowConcurrency: true, noCache: true }); +await this.ctx.storage.put("key", value, { allowUnconfirmed: true, noCache: true }); +``` + +### Storage Options + +| Option | Methods | Effect | Use Case | +|--------|---------|--------|----------| +| `allowConcurrency` | get, list | Skip input gate; allow concurrent requests during read | Read-heavy metrics that don't need strict consistency | +| `noCache` | get, put, list | Skip in-memory cache; always read from disk | Rarely-accessed data or testing storage directly | +| `allowUnconfirmed` | put, delete | Return before write confirms (still protected by output gate) | Non-critical writes where latency matters more than confirmation | + +## Transactions + +```typescript +// Sync (SQL/sync KV only) +this.ctx.storage.transactionSync(() => { + this.sql.exec('UPDATE accounts SET balance = balance - ? WHERE id = ?', 100, 1); + this.sql.exec('UPDATE accounts SET balance = balance + ? WHERE id = ?', 100, 2); + return "result"; +}); + +// Async +await this.ctx.storage.transaction(async () => { + const value = await this.ctx.storage.get("counter"); + await this.ctx.storage.put("counter", value + 1); + if (value > 100) this.ctx.storage.rollback(); // Explicit rollback +}); +``` + +## Point-in-Time Recovery + +```typescript +await this.ctx.storage.getCurrentBookmark(); +await this.ctx.storage.getBookmarkForTime(Date.now() - 2 * 24 * 60 * 60 * 1000); +await this.ctx.storage.onNextSessionRestoreBookmark(bookmark); +this.ctx.abort(); // Restart to apply; bookmarks lexically comparable (earlier < later) +``` + +## Alarms + +```typescript +await this.ctx.storage.setAlarm(Date.now() + 60000); // Timestamp or Date +await this.ctx.storage.getAlarm(); +await this.ctx.storage.deleteAlarm(); + +async alarm() { await this.doScheduledWork(); } +``` + +## Misc + +```typescript +await this.ctx.storage.deleteAll(); // Atomic for SQLite; alarm NOT included +this.ctx.storage.sql.databaseSize; // Bytes +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/do-storage/configuration.md b/plugins/cloudflare/skills/cloudflare/references/do-storage/configuration.md new file mode 100644 index 0000000..18b41bb --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/do-storage/configuration.md @@ -0,0 +1,112 @@ +# DO Storage Configuration + +## SQLite-backed (Recommended) + +**wrangler.jsonc:** +```jsonc +{ + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["Counter", "Session", "RateLimiter"] + } + ] +} +``` + +**Migration lifecycle:** Migrations run once per deployment. Existing DO instances get new storage backend on next invocation. Renaming/removing classes requires `renamed_classes` or `deleted_classes` entries. + +## KV-backed (Legacy) + +**wrangler.jsonc:** +```jsonc +{ + "migrations": [ + { + "tag": "v1", + "new_classes": ["OldCounter"] + } + ] +} +``` + +## TypeScript Setup + +```typescript +export class MyDurableObject extends DurableObject { + sql: SqlStorage; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.sql = ctx.storage.sql; + + // Initialize schema + this.sql.exec(` + CREATE TABLE IF NOT EXISTS users( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT UNIQUE + ); + `); + } +} + +// Binding +interface Env { + MY_DO: DurableObjectNamespace; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const id = env.MY_DO.idFromName('singleton'); + const stub = env.MY_DO.get(id); + + // Modern RPC: call methods directly (recommended) + const result = await stub.someMethod(); + return Response.json(result); + + // Legacy: forward request (still works) + // return stub.fetch(request); + } +} +``` + +## CPU Limits + +```jsonc +{ + "limits": { + "cpu_ms": 300000 // 5 minutes (default 30s) + } +} +``` + +## Location Control + +```typescript +// Jurisdiction (GDPR/FedRAMP) +const euNamespace = env.MY_DO.jurisdiction("eu"); +const id = euNamespace.newUniqueId(); +const stub = euNamespace.get(id); + +// Location hint (best effort) +const stub = env.MY_DO.get(id, { locationHint: "enam" }); +// Hints: wnam, enam, sam, weur, eeur, apac, oc, afr, me +``` + +## Initialization + +```typescript +export class Counter extends DurableObject { + value: number; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + + // Block concurrent requests during init + ctx.blockConcurrencyWhile(async () => { + this.value = (await ctx.storage.get("value")) || 0; + }); + } +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/do-storage/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/do-storage/gotchas.md new file mode 100644 index 0000000..8898f08 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/do-storage/gotchas.md @@ -0,0 +1,150 @@ +# DO Storage Gotchas & Troubleshooting + +## Concurrency Model (CRITICAL) + +Durable Objects use **input/output gates** to prevent race conditions: + +### Input Gates +Block new requests during storage reads from CURRENT request: + +```typescript +// SAFE: Input gate active during await +async increment() { + const val = await this.ctx.storage.get("counter"); // Input gate blocks other requests + await this.ctx.storage.put("counter", val + 1); + return val; +} +``` + +### Output Gates +Hold response until ALL writes from current request confirm: + +```typescript +// SAFE: Output gate waits for put() to confirm before returning response +async increment() { + const val = await this.ctx.storage.get("counter"); + this.ctx.storage.put("counter", val + 1); // No await + return new Response(String(val)); // Response delayed until write confirms +} +``` + +### Write Coalescing +Multiple writes to same key = atomic (last write wins): + +```typescript +// SAFE: All three writes coalesce atomically +this.ctx.storage.put("key", 1); +this.ctx.storage.put("key", 2); +this.ctx.storage.put("key", 3); // Final value: 3 +``` + +### Breaking Gates (DANGER) + +**fetch() breaks input/output gates** → allows request interleaving: + +```typescript +// UNSAFE: fetch() allows another request to interleave +async unsafe() { + const val = await this.ctx.storage.get("counter"); + await fetch("https://api.example.com"); // Gate broken! + await this.ctx.storage.put("counter", val + 1); // Race condition possible +} +``` + +**Solution:** Use `blockConcurrencyWhile()` or `transaction()`: + +```typescript +// SAFE: Block concurrent requests explicitly +async safe() { + return await this.ctx.blockConcurrencyWhile(async () => { + const val = await this.ctx.storage.get("counter"); + await fetch("https://api.example.com"); + await this.ctx.storage.put("counter", val + 1); + return val; + }); +} +``` + +### allowConcurrency Option + +Opt out of input gate for reads that don't need protection: + +```typescript +// Allow concurrent reads (no consistency guarantee) +const val = await this.ctx.storage.get("metrics", { allowConcurrency: true }); +``` + +## Common Errors + +### "Race Condition in Concurrent Calls" + +**Cause:** Multiple concurrent storage operations initiated from same event (e.g., `Promise.all()`) are not protected by input gate +**Solution:** Avoid concurrent storage operations within single event; input gate only serializes requests from different events, not operations within same event + +### "Direct SQL Transaction Statements" + +**Cause:** Using `BEGIN TRANSACTION` directly instead of transaction methods +**Solution:** Use `this.ctx.storage.transactionSync()` for sync operations or `this.ctx.storage.transaction()` for async operations + +### "Async in transactionSync" + +**Cause:** Using async operations inside `transactionSync()` callback +**Solution:** Use async `transaction()` method instead of `transactionSync()` when async operations needed + +### "TypeScript Type Mismatch at Runtime" + +**Cause:** Query doesn't return all fields specified in TypeScript type +**Solution:** Ensure SQL query selects all columns that match the TypeScript type definition + +### "Silent Data Corruption with Large IDs" + +**Cause:** JavaScript numbers have 53-bit precision; SQLite INTEGER is 64-bit +**Symptom:** IDs > 9007199254740991 (Number.MAX_SAFE_INTEGER) silently truncate/corrupt +**Solution:** Store large IDs as TEXT: + +```typescript +// BAD: Snowflake/Twitter IDs will corrupt +this.sql.exec("CREATE TABLE events(id INTEGER PRIMARY KEY)"); +this.sql.exec("INSERT INTO events VALUES (?)", 1234567890123456789n); // Corrupts! + +// GOOD: Store as TEXT +this.sql.exec("CREATE TABLE events(id TEXT PRIMARY KEY)"); +this.sql.exec("INSERT INTO events VALUES (?)", "1234567890123456789"); +``` + +### "Alarm Not Deleted with deleteAll()" + +**Cause:** `deleteAll()` doesn't delete alarms automatically +**Solution:** Call `deleteAlarm()` explicitly before `deleteAll()` to remove alarm + +### "Slow Performance" + +**Cause:** Using async KV API instead of sync API +**Solution:** Use sync KV API (`ctx.storage.kv`) for better performance with simple key-value operations + +### "High Billing from Storage Operations" + +**Cause:** Excessive `rowsRead`/`rowsWritten` or unused objects not cleaned up +**Solution:** Monitor `rowsRead`/`rowsWritten` metrics and ensure unused objects call `deleteAll()` + +### "Durable Object Overloaded" + +**Cause:** Single DO exceeding ~1K req/sec soft limit +**Solution:** Shard across multiple DOs with random IDs or other distribution strategy + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Max columns per table | 100 | SQL limitation | +| Max string/BLOB per row | 2 MB | SQL limitation | +| Max row size | 2 MB | SQL limitation | +| Max SQL statement size | 100 KB | SQL limitation | +| Max SQL parameters | 100 | SQL limitation | +| Max LIKE/GLOB pattern | 50 B | SQL limitation | +| SQLite storage per object | 10 GB | SQLite-backed storage | +| SQLite key+value size | 2 MB | SQLite-backed storage | +| KV storage per object | Unlimited | KV-style storage | +| KV key size | 2 KiB | KV-style storage | +| KV value size | 128 KiB | KV-style storage | +| Request throughput | ~1K req/sec | Soft limit per DO | diff --git a/plugins/cloudflare/skills/cloudflare/references/do-storage/patterns.md b/plugins/cloudflare/skills/cloudflare/references/do-storage/patterns.md new file mode 100644 index 0000000..2885915 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/do-storage/patterns.md @@ -0,0 +1,182 @@ +# DO Storage Patterns & Best Practices + +## Schema Migration + +```typescript +export class MyDurableObject extends DurableObject { + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.sql = ctx.storage.sql; + + // Use SQLite's built-in user_version pragma + const ver = this.sql.exec("PRAGMA user_version").one()?.user_version || 0; + + if (ver === 0) { + this.sql.exec(`CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT)`); + this.sql.exec("PRAGMA user_version = 1"); + } + if (ver === 1) { + this.sql.exec(`ALTER TABLE users ADD COLUMN email TEXT`); + this.sql.exec("PRAGMA user_version = 2"); + } + } +} +``` + +## In-Memory Caching + +```typescript +export class UserCache extends DurableObject { + cache = new Map(); + async getUser(id: string): Promise { + if (this.cache.has(id)) { + const cached = this.cache.get(id); + if (cached) return cached; + } + const user = await this.ctx.storage.get(`user:${id}`); + if (user) this.cache.set(id, user); + return user; + } + async updateUser(id: string, data: Partial) { + const updated = { ...await this.getUser(id), ...data }; + this.cache.set(id, updated); + await this.ctx.storage.put(`user:${id}`, updated); + return updated; + } +} +``` + +## Rate Limiting + +```typescript +export class RateLimiter extends DurableObject { + async checkLimit(key: string, limit: number, window: number): Promise { + const now = Date.now(); + this.sql.exec('DELETE FROM requests WHERE key = ? AND timestamp < ?', key, now - window); + const count = this.sql.exec('SELECT COUNT(*) as count FROM requests WHERE key = ?', key).one().count; + if (count >= limit) return false; + this.sql.exec('INSERT INTO requests (key, timestamp) VALUES (?, ?)', key, now); + return true; + } +} +``` + +## Batch Processing with Alarms + +```typescript +export class BatchProcessor extends DurableObject { + pending: string[] = []; + async addItem(item: string) { + this.pending.push(item); + if (!await this.ctx.storage.getAlarm()) await this.ctx.storage.setAlarm(Date.now() + 5000); + } + async alarm() { + const items = [...this.pending]; + this.pending = []; + this.sql.exec(`INSERT INTO processed_items (item, timestamp) VALUES ${items.map(() => "(?, ?)").join(", ")}`, ...items.flatMap(item => [item, Date.now()])); + } +} +``` + +## Initialization Pattern + +```typescript +export class Counter extends DurableObject { + value: number; + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + ctx.blockConcurrencyWhile(async () => { this.value = (await ctx.storage.get("value")) || 0; }); + } + async increment() { + this.value++; + this.ctx.storage.put("value", this.value); // Don't await (output gate protects) + return this.value; + } +} +``` + +## Safe Counter / Optimized Write + +```typescript +// Input gate blocks other requests +async getUniqueNumber(): Promise { + let val = await this.ctx.storage.get("counter"); + await this.ctx.storage.put("counter", val + 1); + return val; +} + +// No await on write - output gate delays response until write confirms +async increment(): Promise { + let val = await this.ctx.storage.get("counter"); + this.ctx.storage.put("counter", val + 1); + return new Response(String(val)); +} +``` + +## Parent-Child Coordination + +Hierarchical DO pattern where parent manages child DOs: + +```typescript +// Parent DO coordinates children +export class Workspace extends DurableObject { + async createDocument(name: string): Promise { + const docId = crypto.randomUUID(); + const childId = this.env.DOCUMENT.idFromName(`${this.ctx.id.toString()}:${docId}`); + const childStub = this.env.DOCUMENT.get(childId); + await childStub.initialize(name); + + // Track child in parent storage + this.sql.exec('INSERT INTO documents (id, name, created) VALUES (?, ?, ?)', + docId, name, Date.now()); + return docId; + } + + async listDocuments(): Promise { + return this.sql.exec('SELECT id FROM documents').toArray().map(r => r.id); + } +} + +// Child DO +export class Document extends DurableObject { + async initialize(name: string) { + this.sql.exec('CREATE TABLE IF NOT EXISTS content(key TEXT PRIMARY KEY, value TEXT)'); + this.sql.exec('INSERT INTO content VALUES (?, ?)', 'name', name); + } +} +``` + +## Write Coalescing Pattern + +Multiple writes to same key coalesce atomically (last write wins): + +```typescript +async updateMetrics(userId: string, actions: Action[]) { + // All writes coalesce - no await needed + for (const action of actions) { + this.ctx.storage.put(`user:${userId}:lastAction`, action.type); + this.ctx.storage.put(`user:${userId}:count`, + await this.ctx.storage.get(`user:${userId}:count`) + 1); + } + // Output gate ensures all writes confirm before response + return new Response("OK"); +} + +// Atomic batch with SQL +async batchUpdate(items: Item[]) { + this.sql.exec('BEGIN'); + for (const item of items) { + this.sql.exec('INSERT OR REPLACE INTO items VALUES (?, ?)', item.id, item.value); + } + this.sql.exec('COMMIT'); +} +``` + +## Cleanup + +```typescript +async cleanup() { + await this.ctx.storage.deleteAlarm(); // Separate from deleteAll + await this.ctx.storage.deleteAll(); +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/do-storage/testing.md b/plugins/cloudflare/skills/cloudflare/references/do-storage/testing.md new file mode 100644 index 0000000..d348d87 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/do-storage/testing.md @@ -0,0 +1,183 @@ +# DO Storage Testing + +Testing Durable Objects with storage using `vitest-pool-workers`. + +## Setup + +**vitest.config.ts:** +```typescript +import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config"; + +export default defineWorkersConfig({ + test: { + poolOptions: { + workers: { wrangler: { configPath: "./wrangler.toml" } } + } + } +}); +``` + +**package.json:** Add `@cloudflare/vitest-pool-workers` and `vitest` to devDependencies + +## Basic Testing + +```typescript +import { env, runInDurableObject } from "cloudflare:test"; +import { describe, it, expect } from "vitest"; + +describe("Counter DO", () => { + it("increments counter", async () => { + const id = env.COUNTER.idFromName("test"); + const result = await runInDurableObject(env.COUNTER, id, async (instance, state) => { + const val1 = await instance.increment(); + const val2 = await instance.increment(); + return { val1, val2 }; + }); + expect(result.val1).toBe(1); + expect(result.val2).toBe(2); + }); +}); +``` + +## Testing SQL Storage + +```typescript +it("creates and queries users", async () => { + const id = env.USER_MANAGER.idFromName("test"); + await runInDurableObject(env.USER_MANAGER, id, async (instance, state) => { + await instance.createUser("alice@example.com", "Alice"); + const user = await instance.getUser("alice@example.com"); + expect(user).toEqual({ email: "alice@example.com", name: "Alice" }); + }); +}); + +it("handles schema migrations", async () => { + const id = env.USER_MANAGER.idFromName("migration-test"); + await runInDurableObject(env.USER_MANAGER, id, async (instance, state) => { + const version = state.storage.sql.exec( + "SELECT value FROM _meta WHERE key = 'schema_version'" + ).one()?.value; + expect(version).toBe("1"); + }); +}); +``` + +## Testing Alarms + +```typescript +import { runDurableObjectAlarm } from "cloudflare:test"; + +it("processes batch on alarm", async () => { + const id = env.BATCH_PROCESSOR.idFromName("test"); + + // Add items + await runInDurableObject(env.BATCH_PROCESSOR, id, async (instance) => { + await instance.addItem("item1"); + await instance.addItem("item2"); + }); + + // Trigger alarm + await runDurableObjectAlarm(env.BATCH_PROCESSOR, id); + + // Verify processed + await runInDurableObject(env.BATCH_PROCESSOR, id, async (instance, state) => { + const count = state.storage.sql.exec( + "SELECT COUNT(*) as count FROM processed_items" + ).one().count; + expect(count).toBe(2); + }); +}); +``` + +## Testing Concurrency + +```typescript +it("handles concurrent increments safely", async () => { + const id = env.COUNTER.idFromName("concurrent-test"); + + // Parallel increments + const results = await Promise.all([ + runInDurableObject(env.COUNTER, id, (i) => i.increment()), + runInDurableObject(env.COUNTER, id, (i) => i.increment()), + runInDurableObject(env.COUNTER, id, (i) => i.increment()) + ]); + + // All should get unique values + expect(new Set(results).size).toBe(3); + expect(Math.max(...results)).toBe(3); +}); +``` + +## Test Isolation + +```typescript +// Per-test unique IDs +let testId: string; +beforeEach(() => { testId = crypto.randomUUID(); }); + +it("isolated test", async () => { + const id = env.MY_DO.idFromName(testId); + // Uses unique DO instance +}); + +// Cleanup pattern +it("with cleanup", async () => { + const id = env.MY_DO.idFromName("cleanup-test"); + try { + await runInDurableObject(env.MY_DO, id, async (instance) => {}); + } finally { + await runInDurableObject(env.MY_DO, id, async (instance, state) => { + await state.storage.deleteAll(); + }); + } +}); +``` + +## Testing PITR + +```typescript +it("restores from bookmark", async () => { + const id = env.MY_DO.idFromName("pitr-test"); + + // Create checkpoint + const bookmark = await runInDurableObject(env.MY_DO, id, async (instance, state) => { + await state.storage.put("value", 1); + return await state.storage.getCurrentBookmark(); + }); + + // Modify and restore + await runInDurableObject(env.MY_DO, id, async (instance, state) => { + await state.storage.put("value", 2); + await state.storage.onNextSessionRestoreBookmark(bookmark); + state.abort(); + }); + + // Verify restored + await runInDurableObject(env.MY_DO, id, async (instance, state) => { + const value = await state.storage.get("value"); + expect(value).toBe(1); + }); +}); +``` + +## Testing Transactions + +```typescript +it("rolls back on error", async () => { + const id = env.BANK.idFromName("transaction-test"); + + await runInDurableObject(env.BANK, id, async (instance, state) => { + await state.storage.put("balance", 100); + + await expect( + state.storage.transaction(async () => { + await state.storage.put("balance", 50); + throw new Error("Cancel"); + }) + ).rejects.toThrow("Cancel"); + + const balance = await state.storage.get("balance"); + expect(balance).toBe(100); // Rolled back + }); +}); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/durable-objects/README.md b/plugins/cloudflare/skills/cloudflare/references/durable-objects/README.md new file mode 100644 index 0000000..8e96558 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/durable-objects/README.md @@ -0,0 +1,185 @@ +# Cloudflare Durable Objects + +Expert guidance for building stateful applications with Cloudflare Durable Objects. + +## Reading Order + +1. **First time?** Read this overview + Quick Start +2. **Setting up?** See [Configuration](./configuration.md) +3. **Building features?** Use decision trees below → [Patterns](./patterns.md) +4. **Debugging issues?** Check [Gotchas](./gotchas.md) +5. **Deep dive?** [API](./api.md) and [DO Storage](../do-storage/README.md) + +## Overview + +Durable Objects combine compute with storage in globally-unique, strongly-consistent packages: +- **Globally unique instances**: Each DO has unique ID for multi-client coordination +- **Co-located storage**: Fast, strongly-consistent storage with compute +- **Automatic placement**: Objects spawn near first request location +- **Stateful serverless**: In-memory state + persistent storage +- **Single-threaded**: Serial request processing (no race conditions) + +## Rules of Durable Objects + +Critical rules preventing most production issues: + +1. **One alarm per DO** - Schedule multiple events via queue pattern +2. **~1K req/s per DO max** - Shard for higher throughput +3. **Constructor runs every wake** - Keep initialization light; use lazy loading +4. **Hibernation clears memory** - In-memory state lost; persist critical data +5. **Use `ctx.waitUntil()` for cleanup** - Ensures completion after response sent +6. **No setTimeout for persistence** - Use `setAlarm()` for reliable scheduling + +## Core Concepts + +### Class Structure +All DOs extend `DurableObject` base class with constructor receiving `DurableObjectState` (storage, WebSockets, alarms) and `Env` (bindings). + +### Lifecycle States + +``` +[Not Created] → [Active] ⇄ [Hibernated] → [Evicted] + ↓ + [Destroyed] +``` + +- **Not Created**: DO ID exists but instance never spawned +- **Active**: Processing requests, in-memory state valid, billed per GB-hour +- **Hibernated**: WebSocket connections open but zero compute, zero cost +- **Evicted**: Removed from memory; next request triggers cold start +- **Destroyed**: Data deleted via migration or manual deletion + +### Accessing from Workers +Workers use bindings to get stubs, then call RPC methods directly (recommended) or use fetch handler (legacy). + +**RPC vs fetch() decision:** +``` +├─ New project + compat ≥2024-04-03 → RPC (type-safe, simpler) +├─ Need HTTP semantics (headers, status) → fetch() +├─ Proxying requests to DO → fetch() +└─ Legacy compatibility → fetch() +``` + +See [Patterns: RPC vs fetch()](./patterns.md) for examples. + +### ID Generation +- `idFromName()`: Deterministic, named coordination (rate limiting, locks) +- `newUniqueId()`: Random IDs for sharding high-throughput workloads +- `idFromString()`: Derive from existing IDs +- Jurisdiction option: Data locality compliance + +### Storage Options + +**Which storage API?** +``` +├─ Structured data, relations, transactions → SQLite (recommended) +├─ Simple KV on SQLite DO → ctx.storage.kv (sync KV) +└─ Legacy KV-only DO → ctx.storage (async KV) +``` + +- **SQLite** (recommended): Structured data, transactions, 10GB/DO +- **Synchronous KV API**: Simple key-value on SQLite objects +- **Asynchronous KV API**: Legacy/advanced use cases + +See [DO Storage](../do-storage/README.md) for deep dive. + +### Special Features +- **Alarms**: Schedule future execution per-DO (1 per DO - use queue pattern for multiple) +- **WebSocket Hibernation**: Zero-cost idle connections (memory cleared on hibernation) +- **Point-in-Time Recovery**: Restore to any point in 30 days (SQLite only) + +## Quick Start + +```typescript +import { DurableObject } from "cloudflare:workers"; + +export class Counter extends DurableObject { + async increment(): Promise { + const result = this.ctx.storage.sql.exec( + `INSERT INTO counters (id, value) VALUES (1, 1) + ON CONFLICT(id) DO UPDATE SET value = value + 1 + RETURNING value` + ).one(); + return result.value; + } +} + +// Worker access +export default { + async fetch(request: Request, env: Env): Promise { + const id = env.COUNTER.idFromName("global"); + const stub = env.COUNTER.get(id); + const count = await stub.increment(); + return new Response(`Count: ${count}`); + } +}; +``` + +## Decision Trees + +### What do you need? + +``` +├─ Coordinate requests (rate limit, lock, session) +│ → idFromName(identifier) → [Patterns: Rate Limiting/Locks](./patterns.md) +│ +├─ High throughput (>1K req/s) +│ → Sharding with newUniqueId() or hash → [Patterns: Sharding](./patterns.md) +│ +├─ Real-time updates (WebSocket, chat, collab) +│ → WebSocket hibernation + room pattern → [Patterns: Real-time](./patterns.md) +│ +├─ Background work (cleanup, notifications, scheduled tasks) +│ → Alarms + queue pattern (1 alarm/DO) → [Patterns: Multiple Events](./patterns.md) +│ +└─ User sessions with expiration + → Session pattern + alarm cleanup → [Patterns: Session Management](./patterns.md) +``` + +### Which access pattern? + +``` +├─ New project + typed methods → RPC (compat ≥2024-04-03) +├─ Need HTTP semantics → fetch() +├─ Proxying to DO → fetch() +└─ Legacy compat → fetch() +``` + +See [Patterns: RPC vs fetch()](./patterns.md) for examples. + +### Which storage? + +``` +├─ Structured data, SQL queries, transactions → SQLite (recommended) +├─ Simple KV on SQLite DO → ctx.storage.kv (sync API) +└─ Legacy KV-only DO → ctx.storage (async API) +``` + +See [DO Storage](../do-storage/README.md) for complete guide. + +## Essential Commands + +```bash +npx wrangler dev # Local dev with DOs +npx wrangler dev --remote # Test against prod DOs +npx wrangler deploy # Deploy + auto-apply migrations +``` + +## Resources + +**Docs**: https://developers.cloudflare.com/durable-objects/ +**API Reference**: https://developers.cloudflare.com/durable-objects/api/ +**Examples**: https://developers.cloudflare.com/durable-objects/examples/ + +## In This Reference + +- **[Configuration](./configuration.md)** - wrangler.jsonc setup, migrations, bindings, environments +- **[API](./api.md)** - Class structure, ctx methods, alarms, WebSocket hibernation +- **[Patterns](./patterns.md)** - Sharding, rate limiting, locks, real-time, sessions +- **[Gotchas](./gotchas.md)** - Limits, hibernation caveats, common errors + +## See Also + +- **[DO Storage](../do-storage/README.md)** - SQLite, KV, transactions (detailed storage guide) +- **[Workers](../workers/README.md)** - Core Workers runtime features +- **[WebSockets](../websockets/README.md)** - WebSocket APIs and patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/durable-objects/api.md b/plugins/cloudflare/skills/cloudflare/references/durable-objects/api.md new file mode 100644 index 0000000..89c7e4d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/durable-objects/api.md @@ -0,0 +1,187 @@ +# Durable Objects API + +## Class Structure + +```typescript +import { DurableObject } from "cloudflare:workers"; + +export class MyDO extends DurableObject { + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + // Runs on EVERY wake - keep light! + } + + // RPC methods (called directly from worker) + async myMethod(arg: string): Promise { return arg; } + + // fetch handler (legacy/HTTP semantics) + async fetch(req: Request): Promise { /* ... */ } + + // Lifecycle handlers + async alarm() { /* alarm fired */ } + async webSocketMessage(ws: WebSocket, msg: string | ArrayBuffer) { /* ... */ } + async webSocketClose(ws: WebSocket, code: number, reason: string, wasClean: boolean) { /* ... */ } + async webSocketError(ws: WebSocket, error: unknown) { /* ... */ } +} +``` + +## DurableObjectState Context Methods + +### Concurrency Control + +```typescript +// Complete work after response sent (e.g., cleanup, logging) +this.ctx.waitUntil(promise: Promise): void + +// Critical section - blocks all other requests until complete +await this.ctx.blockConcurrencyWhile(async () => { + // No other requests processed during this block + // Use for initialization or critical operations +}) +``` + +**When to use:** +- `waitUntil()`: Background cleanup, logging, non-critical work after response +- `blockConcurrencyWhile()`: First-time init, schema migration, critical state setup + +### Lifecycle + +```typescript +this.ctx.id // DurableObjectId of this instance +this.ctx.abort() // Force eviction (use after PITR restore to reload state) +``` + +### Storage Access + +```typescript +this.ctx.storage.sql // SQLite API (recommended) +this.ctx.storage.kv // Sync KV API (SQLite DOs only) +this.ctx.storage // Async KV API (legacy/KV-only DOs) +``` + +See **[DO Storage](../do-storage/README.md)** for complete storage API reference. + +### WebSocket Management + +```typescript +this.ctx.acceptWebSocket(ws: WebSocket, tags?: string[]) // Enable hibernation +this.ctx.getWebSockets(tag?: string): WebSocket[] // Get by tag or all +this.ctx.getTags(ws: WebSocket): string[] // Get tags for connection +``` + +### Alarms + +```typescript +await this.ctx.storage.setAlarm(timestamp: number | Date) // Schedule (overwrites existing) +await this.ctx.storage.getAlarm(): number | null // Get next alarm time +await this.ctx.storage.deleteAlarm(): void // Cancel alarm +``` + +**Limit:** 1 alarm per DO. Use queue pattern for multiple events (see [Patterns](./patterns.md)). + +## Storage APIs + +For detailed storage documentation including SQLite queries, KV operations, transactions, and Point-in-Time Recovery, see **[DO Storage](../do-storage/README.md)**. + +Quick reference: + +```typescript +// SQLite (recommended) +this.ctx.storage.sql.exec("SELECT * FROM users WHERE id = ?", userId).one() + +// Sync KV (SQLite DOs only) +this.ctx.storage.kv.get("key") + +// Async KV (legacy) +await this.ctx.storage.get("key") +``` + +## Alarms + +Schedule future work that survives eviction: + +```typescript +// Set alarm (overwrites any existing alarm) +await this.ctx.storage.setAlarm(Date.now() + 3600000) // 1 hour from now +await this.ctx.storage.setAlarm(new Date("2026-02-01")) // Absolute time + +// Check next alarm +const nextRun = await this.ctx.storage.getAlarm() // null if none + +// Cancel alarm +await this.ctx.storage.deleteAlarm() + +// Handler called when alarm fires +async alarm() { + // Runs once alarm triggers + // DO wakes from hibernation if needed + // Use for cleanup, notifications, scheduled tasks +} +``` + +**Limitations:** +- 1 alarm per DO maximum +- Overwrites previous alarm when set +- Use queue pattern for multiple scheduled events (see [Patterns](./patterns.md)) + +**Reliability:** +- Alarms survive DO eviction/restart +- Cloudflare retries failed alarms automatically +- Not guaranteed exactly-once (handle idempotently) + +## WebSocket Hibernation + +Hibernation allows DOs with open WebSocket connections to consume zero compute/memory until message arrives. + +```typescript +async fetch(req: Request): Promise { + const [client, server] = Object.values(new WebSocketPair()); + this.ctx.acceptWebSocket(server, ["room:123"]); // Tags for filtering + server.serializeAttachment({ userId: "abc" }); // Persisted metadata + return new Response(null, { status: 101, webSocket: client }); +} + +// Called when message arrives (DO wakes from hibernation) +async webSocketMessage(ws: WebSocket, msg: string | ArrayBuffer) { + const data = ws.deserializeAttachment(); // Retrieve metadata + for (const c of this.ctx.getWebSockets("room:123")) c.send(msg); +} + +// Called on close (optional handler) +async webSocketClose(ws: WebSocket, code: number, reason: string, wasClean: boolean) { + // Cleanup logic, remove from lists, etc. +} + +// Called on error (optional handler) +async webSocketError(ws: WebSocket, error: unknown) { + console.error("WebSocket error:", error); + // Handle error, close connection, etc. +} +``` + +**Key concepts:** +- **Auto-hibernation:** DO hibernates when no active requests/alarms +- **Zero cost:** Hibernated DOs incur no charges while preserving connections +- **Memory cleared:** All in-memory state lost on hibernation +- **Attachment persistence:** Use `serializeAttachment()` for per-connection metadata that survives hibernation +- **Tags for filtering:** Group connections by room/channel/user for targeted broadcasts + +**Handler lifecycle:** +- `webSocketMessage`: DO wakes, processes message, may hibernate after +- `webSocketClose`: Called when client closes (optional - implement for cleanup) +- `webSocketError`: Called on connection error (optional - implement for error handling) + +**Metadata persistence:** +```typescript +// Store connection metadata (survives hibernation) +ws.serializeAttachment({ userId: "abc", room: "lobby" }) + +// Retrieve after hibernation +const { userId, room } = ws.deserializeAttachment() +``` + +## See Also + +- **[DO Storage](../do-storage/README.md)** - Complete storage API reference +- **[Patterns](./patterns.md)** - Real-world usage patterns +- **[Gotchas](./gotchas.md)** - Hibernation caveats and limits diff --git a/plugins/cloudflare/skills/cloudflare/references/durable-objects/configuration.md b/plugins/cloudflare/skills/cloudflare/references/durable-objects/configuration.md new file mode 100644 index 0000000..651599a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/durable-objects/configuration.md @@ -0,0 +1,160 @@ +# Durable Objects Configuration + +## Basic Setup + +```jsonc +{ + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use latest; ≥2024-04-03 for RPC + "durable_objects": { + "bindings": [ + { + "name": "MY_DO", // Env binding name + "class_name": "MyDO" // Class exported from this worker + }, + { + "name": "EXTERNAL", // Access DO from another worker + "class_name": "ExternalDO", + "script_name": "other-worker" + } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["MyDO"] } // Prefer SQLite + ] +} +``` + +## Binding Options + +```jsonc +{ + "name": "BINDING_NAME", + "class_name": "ClassName", + "script_name": "other-worker", // Optional: external DO + "environment": "production" // Optional: isolate by env +} +``` + +## Jurisdiction (Data Locality) + +Specify jurisdiction at ID creation for data residency compliance: + +```typescript +// EU data residency +const id = env.MY_DO.idFromName("user:123", { jurisdiction: "eu" }) + +// Available jurisdictions +const jurisdictions = ["eu", "fedramp"] // More may be added + +// All operations on this DO stay within jurisdiction +const stub = env.MY_DO.get(id) +await stub.someMethod() // Data stays in EU +``` + +**Key points:** +- Set at ID creation time, immutable afterward +- DO instance physically located within jurisdiction +- Storage and compute guaranteed within boundary +- Use for GDPR, FedRAMP, other compliance requirements +- No cross-jurisdiction access (requests fail if DO in different jurisdiction) + +## Migrations + +```jsonc +{ + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["MyDO"] }, // Create SQLite (recommended) + // { "tag": "v1", "new_classes": ["MyDO"] }, // Create KV (paid only) + { "tag": "v2", "renamed_classes": [{ "from": "Old", "to": "New" }] }, + { "tag": "v3", "transferred_classes": [{ "from": "Src", "from_script": "old", "to": "Dest" }] }, + { "tag": "v4", "deleted_classes": ["Obsolete"] } // Destroys ALL data! + ] +} +``` + +**Migration rules:** +- Tags must be unique and sequential (v1, v2, v3...) +- No rollback supported (test with `--dry-run` first) +- Auto-applied on deploy +- `new_sqlite_classes` recommended over `new_classes` (SQLite vs KV) +- `deleted_classes` immediately destroys ALL data (irreversible) + +## Environment Isolation + +Separate DO namespaces per environment (staging/production have distinct object instances): + +```jsonc +{ + "durable_objects": { + "bindings": [{ "name": "MY_DO", "class_name": "MyDO" }] + }, + "env": { + "production": { + "durable_objects": { + "bindings": [ + { "name": "MY_DO", "class_name": "MyDO", "environment": "production" } + ] + } + } + } +} +``` + +Deploy: `npx wrangler deploy --env production` + +## Limits & Settings + +```jsonc +{ + "limits": { + "cpu_ms": 300000 // Max CPU time: 30s default, 300s max + } +} +``` + +See [Gotchas](./gotchas.md) for complete limits table. + +## Types + +```typescript +import { DurableObject } from "cloudflare:workers"; + +interface Env { + MY_DO: DurableObjectNamespace; +} + +export class MyDO extends DurableObject {} + +type DurableObjectNamespace = { + newUniqueId(options?: { jurisdiction?: string }): DurableObjectId; + idFromName(name: string): DurableObjectId; + idFromString(id: string): DurableObjectId; + get(id: DurableObjectId): DurableObjectStub; +}; +``` + +## Commands + +```bash +# Development +npx wrangler dev # Local dev +npx wrangler dev --remote # Test against production DOs + +# Deployment +npx wrangler deploy # Deploy + auto-apply migrations +npx wrangler deploy --dry-run # Validate migrations without deploying +npx wrangler deploy --env production + +# Management +npx wrangler durable-objects list # List namespaces +npx wrangler durable-objects info # Inspect specific DO +npx wrangler durable-objects delete # Delete DO (destroys data) +``` + +## See Also + +- **[API](./api.md)** - DurableObjectState and lifecycle handlers +- **[Patterns](./patterns.md)** - Multi-environment patterns +- **[Gotchas](./gotchas.md)** - Migration caveats, limits diff --git a/plugins/cloudflare/skills/cloudflare/references/durable-objects/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/durable-objects/gotchas.md new file mode 100644 index 0000000..72495f9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/durable-objects/gotchas.md @@ -0,0 +1,197 @@ +# Durable Objects Gotchas + +## Common Errors + +### "Hibernation Cleared My In-Memory State" + +**Problem:** Variables lost after hibernation +**Cause:** DO auto-hibernates when idle; in-memory state not persisted +**Solution:** Use `ctx.storage` for critical data, `ws.serializeAttachment()` for per-connection metadata + +```typescript +// ❌ Wrong - lost on hibernation +private userCount = 0; +async webSocketMessage(ws: WebSocket, msg: string) { + this.userCount++; // Lost! +} + +// ✅ Right - persisted +async webSocketMessage(ws: WebSocket, msg: string) { + const count = this.ctx.storage.kv.get("userCount") || 0; + this.ctx.storage.kv.put("userCount", count + 1); +} +``` + +### "setTimeout Didn't Fire After Restart" + +**Problem:** Scheduled work lost on eviction +**Cause:** `setTimeout` in-memory only; eviction clears timers +**Solution:** Use `ctx.storage.setAlarm()` for reliable scheduling + +```typescript +// ❌ Wrong - lost on eviction +setTimeout(() => this.cleanup(), 3600000); + +// ✅ Right - survives eviction +await this.ctx.storage.setAlarm(Date.now() + 3600000); +async alarm() { await this.cleanup(); } +``` + +### "Constructor Runs on Every Wake" + +**Problem:** Expensive init logic slows all requests +**Cause:** Constructor runs on every wake (first request after eviction OR after hibernation) +**Solution:** Lazy initialization or cache in storage + +**Critical understanding:** Constructor runs in two scenarios: +1. **Cold start** - DO evicted from memory, first request creates new instance +2. **Wake from hibernation** - DO with WebSockets hibernated, message/alarm wakes it + +```typescript +// ❌ Wrong - expensive on every wake +constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.heavyData = this.loadExpensiveData(); // Slow! +} + +// ✅ Right - lazy load +private heavyData?: HeavyData; +private getHeavyData() { + if (!this.heavyData) this.heavyData = this.loadExpensiveData(); + return this.heavyData; +} +``` + +### "Durable Object Overloaded (503 errors)" + +**Problem:** 503 errors under load +**Cause:** Single DO exceeding ~1K req/s throughput limit +**Solution:** Shard across multiple DOs (see [Patterns: Sharding](./patterns.md)) + +### "Storage Quota Exceeded (Write failures)" + +**Problem:** Write operations failing +**Cause:** DO storage exceeding 10GB limit or account quota +**Solution:** Cleanup with alarms, use `deleteAll()` for old data, upgrade plan + +### "CPU Time Exceeded (Terminated)" + +**Problem:** Request terminated mid-execution +**Cause:** Processing exceeding 30s CPU time default limit +**Solution:** Increase `limits.cpu_ms` in wrangler.jsonc (max 300s) or chunk work + +### "WebSockets Disconnect on Eviction" + +**Problem:** Connections drop unexpectedly +**Cause:** DO evicted from memory without hibernation API +**Solution:** Use WebSocket hibernation handlers + client reconnection logic + +### "Migration Failed (Deploy error)" + +**Cause:** Non-unique tags, non-sequential tags, or invalid class names in migration +**Solution:** Check tag uniqueness/sequential ordering and verify class names are correct + +### "RPC Method Not Found" + +**Cause:** compatibility_date < 2024-04-03 preventing RPC usage +**Solution:** Update compatibility_date to >= 2024-04-03 or use fetch() instead of RPC + +### "Only One Alarm Allowed" + +**Cause:** Need multiple scheduled tasks but only one alarm supported per DO +**Solution:** Use event queue pattern to schedule multiple tasks with single alarm + +### "Race Condition Despite Single-Threading" + +**Problem:** Concurrent requests see inconsistent state +**Cause:** Async operations allow request interleaving (await = yield point) +**Solution:** Use `blockConcurrencyWhile()` for critical sections or atomic storage ops + +```typescript +// ❌ Wrong - race condition +async incrementCounter() { + const count = await this.ctx.storage.get("count") || 0; + // ⚠️ Another request could execute here during await + await this.ctx.storage.put("count", count + 1); +} + +// ✅ Right - atomic operation +async incrementCounter() { + return this.ctx.storage.sql.exec( + "INSERT INTO counters (id, value) VALUES (1, 1) ON CONFLICT(id) DO UPDATE SET value = value + 1 RETURNING value" + ).one().value; +} + +// ✅ Right - explicit locking +async criticalOperation() { + await this.ctx.blockConcurrencyWhile(async () => { + const count = await this.ctx.storage.get("count") || 0; + await this.ctx.storage.put("count", count + 1); + }); +} +``` + +### "Migration Rollback Not Supported" + +**Cause:** Attempting to rollback a migration after deployment +**Solution:** Test with `--dry-run` before deploying; migrations cannot be rolled back + +### "deleted_classes Destroys Data" + +**Problem:** Migration deleted all data +**Cause:** `deleted_classes` migration immediately destroys all DO instances and data +**Solution:** Test with `--dry-run`; use `transferred_classes` to preserve data during moves + +### "Cold Starts Are Slow" + +**Problem:** First request after eviction takes longer +**Cause:** DO constructor + initial storage access on cold start +**Solution:** Expected behavior; optimize constructor, use connection pooling in clients, consider warming strategy for critical DOs + +```typescript +// Warming strategy (periodically ping critical DOs) +export default { + async scheduled(event: ScheduledEvent, env: Env) { + const criticalIds = ["auth", "sessions", "locks"]; + await Promise.all(criticalIds.map(name => { + const id = env.MY_DO.idFromName(name); + const stub = env.MY_DO.get(id); + return stub.ping(); // Keep warm + })); + } +}; +``` + +## Limits + +| Limit | Free | Paid | Notes | +|-------|------|------|-------| +| SQLite storage per DO | 10 GB | 10 GB | Per Durable Object instance | +| SQLite total storage | 5 GB | Unlimited | Account-wide quota | +| Key+value size | 2 MB | 2 MB | Single KV pair (SQLite/async) | +| CPU time default | 30s | 30s | Per request; configurable | +| CPU time max | 300s | 300s | Set via `limits.cpu_ms` | +| DO classes | 100 | 500 | Distinct DO class definitions | +| SQL columns | 100 | 100 | Per table | +| SQL statement size | 100 KB | 100 KB | Max SQL query size | +| WebSocket message size | 32 MiB | 32 MiB | Per message | +| Request throughput | ~1K req/s | ~1K req/s | Per DO (soft limit - shard for more) | +| Alarms per DO | 1 | 1 | Use queue pattern for multiple events | +| Total DOs | Unlimited | Unlimited | Create as many instances as needed | +| WebSockets | Unlimited | Unlimited | Within 128MB memory limit per DO | +| Memory per DO | 128 MB | 128 MB | In-memory state + WebSocket buffers | + +## Hibernation Caveats + +1. **Memory cleared** - All in-memory variables lost; reconstruct from storage or `deserializeAttachment()` +2. **Constructor reruns** - Runs on wake; avoid expensive operations, use lazy initialization +3. **No guarantees** - DO may evict instead of hibernate; design for both +4. **Attachment limit** - `serializeAttachment()` data must be JSON-serializable, keep small +5. **Alarm wakes DO** - Alarm prevents hibernation until handler completes +6. **WebSocket state not automatic** - Must explicitly persist with `serializeAttachment()` or storage + +## See Also + +- **[Patterns](./patterns.md)** - Workarounds for common limitations +- **[API](./api.md)** - Storage limits and quotas +- **[Configuration](./configuration.md)** - Setting CPU limits diff --git a/plugins/cloudflare/skills/cloudflare/references/durable-objects/patterns.md b/plugins/cloudflare/skills/cloudflare/references/durable-objects/patterns.md new file mode 100644 index 0000000..d91f382 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/durable-objects/patterns.md @@ -0,0 +1,201 @@ +# Durable Objects Patterns + +## When to Use Which Pattern + +| Need | Pattern | ID Strategy | +|------|---------|-------------| +| Rate limit per user/IP | Rate Limiting | `idFromName(identifier)` | +| Mutual exclusion | Distributed Lock | `idFromName(resource)` | +| >1K req/s throughput | Sharding | `newUniqueId()` or hash | +| Real-time updates | WebSocket Collab | `idFromName(room)` | +| User sessions | Session Management | `idFromName(sessionId)` | +| Background cleanup | Alarm-based | Any | + +## RPC vs fetch() + +**RPC** (compat ≥2024-04-03): Type-safe, simpler, default for new projects +**fetch()**: Legacy compat, HTTP semantics, proxying + +```typescript +const count = await stub.increment(); // RPC +const count = await (await stub.fetch(req)).json(); // fetch() +``` + +## Sharding (High Throughput) + +Single DO ~1K req/s max. Shard for higher throughput: + +```typescript +export default { + async fetch(req: Request, env: Env): Promise { + const userId = new URL(req.url).searchParams.get("user"); + const hash = hashCode(userId) % 100; // 100 shards + const id = env.COUNTER.idFromName(`shard:${hash}`); + return env.COUNTER.get(id).fetch(req); + } +}; + +function hashCode(str: string): number { + let hash = 0; + for (let i = 0; i < str.length; i++) hash = ((hash << 5) - hash) + str.charCodeAt(i); + return Math.abs(hash); +} +``` + +**Decisions:** +- **Shard count**: 10-1000 typical (start with 100, measure, adjust) +- **Shard key**: User ID, IP, session - must distribute evenly (use hash) +- **Aggregation**: Coordinator DO or external system (D1, R2) + +## Rate Limiting + +```typescript +async checkLimit(key: string, limit: number, windowMs: number): Promise { + const req = this.ctx.storage.sql.exec("SELECT COUNT(*) as count FROM requests WHERE key = ? AND timestamp > ?", key, Date.now() - windowMs).one(); + if (req.count >= limit) return false; + this.ctx.storage.sql.exec("INSERT INTO requests (key, timestamp) VALUES (?, ?)", key, Date.now()); + return true; +} +``` + +## Distributed Lock + +```typescript +private held = false; +async acquire(timeoutMs = 5000): Promise { + if (this.held) return false; + this.held = true; + await this.ctx.storage.setAlarm(Date.now() + timeoutMs); + return true; +} +async release() { this.held = false; await this.ctx.storage.deleteAlarm(); } +async alarm() { this.held = false; } // Auto-release on timeout +``` + +## Hibernation-Aware Pattern + +Preserve state across hibernation: + +```typescript +async fetch(req: Request): Promise { + const [client, server] = Object.values(new WebSocketPair()); + const userId = new URL(req.url).searchParams.get("user"); + server.serializeAttachment({ userId }); // Survives hibernation + this.ctx.acceptWebSocket(server, ["room:lobby"]); + server.send(JSON.stringify({ type: "init", state: this.ctx.storage.kv.get("state") })); + return new Response(null, { status: 101, webSocket: client }); +} + +async webSocketMessage(ws: WebSocket, msg: string) { + const { userId } = ws.deserializeAttachment(); // Retrieve after wake + const state = this.ctx.storage.kv.get("state") || {}; + state[userId] = JSON.parse(msg); + this.ctx.storage.kv.put("state", state); + for (const c of this.ctx.getWebSockets("room:lobby")) c.send(msg); +} +``` + +## Real-time Collaboration + +Broadcast updates to all connected clients: + +```typescript +async webSocketMessage(ws: WebSocket, msg: string) { + const data = JSON.parse(msg); + this.ctx.storage.kv.put("doc", data.content); // Persist + for (const c of this.ctx.getWebSockets()) if (c !== ws) c.send(msg); // Broadcast +} +``` + +### WebSocket Reconnection + +**Client-side** (exponential backoff): +```typescript +class ResilientWS { + private delay = 1000; + connect(url: string) { + const ws = new WebSocket(url); + ws.onclose = () => setTimeout(() => { + this.connect(url); + this.delay = Math.min(this.delay * 2, 30000); + }, this.delay); + } +} +``` + +**Server-side** (cleanup on close): +```typescript +async webSocketClose(ws: WebSocket, code: number, reason: string, wasClean: boolean) { + const { userId } = ws.deserializeAttachment(); + this.ctx.storage.sql.exec("UPDATE users SET online = false WHERE id = ?", userId); + for (const c of this.ctx.getWebSockets()) c.send(JSON.stringify({ type: "user_left", userId })); +} +``` + +## Session Management + +```typescript +async createSession(userId: string, data: object): Promise { + const id = crypto.randomUUID(), exp = Date.now() + 86400000; + this.ctx.storage.sql.exec("INSERT INTO sessions VALUES (?, ?, ?, ?)", id, userId, JSON.stringify(data), exp); + await this.ctx.storage.setAlarm(exp); + return id; +} + +async getSession(id: string): Promise { + const row = this.ctx.storage.sql.exec("SELECT data FROM sessions WHERE id = ? AND expires_at > ?", id, Date.now()).one(); + return row ? JSON.parse(row.data) : null; +} + +async alarm() { this.ctx.storage.sql.exec("DELETE FROM sessions WHERE expires_at <= ?", Date.now()); } +``` + +## Multiple Events (Single Alarm) + +Queue pattern to schedule multiple events: + +```typescript +async scheduleEvent(id: string, runAt: number) { + await this.ctx.storage.put(`event:${id}`, { id, runAt }); + const curr = await this.ctx.storage.getAlarm(); + if (!curr || runAt < curr) await this.ctx.storage.setAlarm(runAt); +} + +async alarm() { + const events = await this.ctx.storage.list({ prefix: "event:" }), now = Date.now(); + let next = null; + for (const [key, ev] of events) { + if (ev.runAt <= now) { + await this.processEvent(ev); + await this.ctx.storage.delete(key); + } else if (!next || ev.runAt < next) next = ev.runAt; + } + if (next) await this.ctx.storage.setAlarm(next); +} +``` + +## Graceful Cleanup + +Use `ctx.waitUntil()` to complete work after response: + +```typescript +async myMethod() { + const response = { success: true }; + this.ctx.waitUntil(this.ctx.storage.sql.exec("DELETE FROM old_data WHERE timestamp < ?", cutoff)); + return response; +} +``` + +## Best Practices + +- **Design**: Use `idFromName()` for coordination, `newUniqueId()` for sharding, minimize constructor work +- **Storage**: Prefer SQLite, batch with transactions, set alarms for cleanup, use PITR before risky ops +- **Performance**: ~1K req/s per DO max - shard for more, cache in memory, use alarms for deferred work +- **Reliability**: Handle 503 with retry+backoff, design for cold starts, test migrations with `--dry-run` +- **Security**: Validate inputs in Workers, rate limit DO creation, use jurisdiction for compliance + +## See Also + +- **[API](./api.md)** - ctx methods, WebSocket handlers +- **[Gotchas](./gotchas.md)** - Hibernation caveats, common errors +- **[DO Storage](../do-storage/README.md)** - Storage patterns and transactions diff --git a/plugins/cloudflare/skills/cloudflare/references/email-routing/README.md b/plugins/cloudflare/skills/cloudflare/references/email-routing/README.md new file mode 100644 index 0000000..7fa902e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-routing/README.md @@ -0,0 +1,89 @@ +# Cloudflare Email Routing Skill Reference + +## Overview + +Cloudflare Email Routing enables custom email addresses for your domain that route to verified destination addresses. It's free, privacy-focused (no storage/access), and includes Email Workers for programmatic email processing. + +**Available to all Cloudflare customers using Cloudflare as authoritative nameserver.** + +## Quick Start + +```typescript +// Basic email handler +export default { + async email(message, env, ctx) { + // CRITICAL: Must consume stream before response + const parser = new PostalMime.default(); + const email = await parser.parse(await message.raw.arrayBuffer()); + + // Process email + console.log(`From: ${message.from}, Subject: ${email.subject}`); + + // Forward or reject + await message.forward("verified@destination.com"); + } +} satisfies ExportedHandler; +``` + +## Reading Order + +**Start here based on your goal:** + +1. **New to Email Routing?** → [configuration.md](configuration.md) → [patterns.md](patterns.md) +2. **Adding Workers?** → [api.md](api.md) § Worker Runtime API → [patterns.md](patterns.md) +3. **Sending emails?** → [api.md](api.md) § SendEmail Binding +4. **Managing via API?** → [api.md](api.md) § REST API Operations +5. **Debugging issues?** → [gotchas.md](gotchas.md) + +## Decision Tree + +``` +Need to receive emails? +├─ Simple forwarding only? → Dashboard rules (configuration.md) +├─ Complex logic/filtering? → Email Workers (api.md + patterns.md) +└─ Parse attachments/body? → postal-mime library (patterns.md § Parse Email) + +Need to send emails? +├─ From Worker? → SendEmail binding (api.md § SendEmail) +└─ From external app? → Use external SMTP/API service + +Having issues? +├─ Email not arriving? → gotchas.md § Mail Authentication +├─ Worker crashing? → gotchas.md § Stream Consumption +└─ Forward failing? → gotchas.md § Destination Verification +``` + +## Key Concepts + +**Routing Rules**: Pattern-based forwarding configured via Dashboard/API. Simple but limited. + +**Email Workers**: Custom TypeScript handlers with full email access. Handles complex logic, parsing, storage, rejection. + +**SendEmail Binding**: Outbound email API for Workers. Transactional email only (no marketing/bulk). + +**ForwardableEmailMessage**: Runtime interface for incoming emails. Provides headers, raw stream, forward/reject methods. + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, wrangler config +- **[api.md](api.md)** - REST API + Worker runtime API + types +- **[patterns.md](patterns.md)** - Common patterns with working examples +- **[gotchas.md](gotchas.md)** - Critical pitfalls, troubleshooting, limits + +## Architecture + +``` +Internet → MX Records → Cloudflare Email Routing + ├─ Routing Rules (dashboard) + └─ Email Worker (your code) + ├─ Forward to destination + ├─ Reject with reason + ├─ Store in R2/KV/D1 + └─ Send outbound (SendEmail) +``` + +## See Also + +- [Cloudflare Docs: Email Routing](https://developers.cloudflare.com/email-routing/) +- [Cloudflare Docs: Email Workers](https://developers.cloudflare.com/email-routing/email-workers/) +- [postal-mime npm package](https://www.npmjs.com/package/postal-mime) diff --git a/plugins/cloudflare/skills/cloudflare/references/email-routing/api.md b/plugins/cloudflare/skills/cloudflare/references/email-routing/api.md new file mode 100644 index 0000000..33b8bf0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-routing/api.md @@ -0,0 +1,195 @@ +# Email Routing API Reference + +## Worker Runtime API + +### Email Handler Interface + +```typescript +interface ExportedHandler { + email?(message: ForwardableEmailMessage, env: Env, ctx: ExecutionContext): void | Promise; +} +``` + +### ForwardableEmailMessage + +Main interface for incoming emails: + +```typescript +interface ForwardableEmailMessage { + readonly from: string; // Envelope sender (e.g., "sender@example.com") + readonly to: string; // Envelope recipient (e.g., "you@yourdomain.com") + readonly headers: Headers; // Web API Headers object + readonly raw: ReadableStream; // Raw MIME message stream + + setReject(reason: string): void; + forward(rcptTo: string, headers?: Headers): Promise; +} +``` + +**Key Properties:** + +| Property | Type | Description | +|----------|------|-------------| +| `from` | `string` | Envelope sender (MAIL FROM), not header From | +| `to` | `string` | Envelope recipient (RCPT TO), not header To | +| `headers` | `Headers` | Email headers (Subject, From, To, etc.) | +| `raw` | `ReadableStream` | Raw MIME message (consume once only) | + +**Methods:** + +- `setReject(reason)`: Reject email with bounce message +- `forward(rcptTo, headers?)`: Forward to verified destination, optionally add headers + +### Headers Object + +Standard Web API Headers interface: + +```typescript +// Access headers +const subject = message.headers.get("subject"); +const from = message.headers.get("from"); +const messageId = message.headers.get("message-id"); + +// Check spam score +const spamScore = parseFloat(message.headers.get("x-cf-spamh-score") || "0"); +if (spamScore > 5) { + message.setReject("Spam detected"); +} +``` + +### Common Headers + +`subject`, `from`, `to`, `x-cf-spamh-score` (spam score), `message-id` (deduplication), `dkim-signature` (auth) + +### Envelope vs Header Addresses + +**Critical distinction:** + +```typescript +// Envelope addresses (routing, auth checks) +message.from // "bounce@sender.com" (actual sender) +message.to // "you@yourdomain.com" (your address) + +// Header addresses (display, user-facing) +message.headers.get("from") // "Alice " +message.headers.get("to") // "Bob " +``` + +**Use envelope addresses for:** +- Authentication/SPF checks +- Routing decisions +- Bounce handling + +**Use header addresses for:** +- Display to users +- Reply-To logic +- User-facing filtering + +## SendEmail Binding + +Outbound email API for transactional messages. + +### Configuration + +```jsonc +// wrangler.jsonc +{ + "send_email": [ + { "name": "EMAIL" } + ] +} +``` + +### TypeScript Types + +```typescript +interface Env { + EMAIL: SendEmail; +} + +interface SendEmail { + send(message: EmailMessage): Promise; +} + +interface EmailMessage { + from: string | { name?: string; email: string }; + to: string | { name?: string; email: string } | Array; + subject: string; + text?: string; + html?: string; + headers?: Headers; + reply_to?: string | { name?: string; email: string }; +} +``` + +### Send Email Example + +```typescript +interface Env { + EMAIL: SendEmail; +} + +export default { + async fetch(request, env, ctx): Promise { + await env.EMAIL.send({ + from: { name: "Acme Corp", email: "noreply@yourdomain.com" }, + to: [ + { name: "Alice", email: "alice@example.com" }, + "bob@example.com" + ], + subject: "Your order #12345 has shipped", + text: "Track your package at: https://track.example.com/12345", + html: "

Track your package at: View tracking

", + reply_to: { name: "Support", email: "support@yourdomain.com" } + }); + + return new Response("Email sent"); + } +} satisfies ExportedHandler; +``` + +### SendEmail Constraints + +- **From address**: Must be on verified domain (your domain with Email Routing enabled) +- **Volume limits**: Transactional only, no bulk/marketing email +- **Rate limits**: 100 emails/minute on Free plan, higher on Paid +- **No attachments**: Use links to hosted files instead +- **No DKIM control**: Cloudflare signs automatically + +## REST API Operations + +Base URL: `https://api.cloudflare.com/client/v4` + +### Authentication + +```bash +curl -H "Authorization: Bearer $API_TOKEN" https://api.cloudflare.com/client/v4/... +``` + +### Key Endpoints + +| Operation | Method | Endpoint | +|-----------|--------|----------| +| Enable routing | POST | `/zones/{zone_id}/email/routing/enable` | +| Disable routing | POST | `/zones/{zone_id}/email/routing/disable` | +| List rules | GET | `/zones/{zone_id}/email/routing/rules` | +| Create rule | POST | `/zones/{zone_id}/email/routing/rules` | +| Verify destination | POST | `/zones/{zone_id}/email/routing/addresses` | +| List destinations | GET | `/zones/{zone_id}/email/routing/addresses` | + +### Create Routing Rule Example + +```bash +curl -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/email/routing/rules" \ + -H "Authorization: Bearer $API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "enabled": true, + "name": "Forward sales", + "matchers": [{"type": "literal", "field": "to", "value": "sales@yourdomain.com"}], + "actions": [{"type": "forward", "value": ["alice@company.com"]}], + "priority": 0 + }' +``` + +Matcher types: `literal` (exact match), `all` (catch-all). diff --git a/plugins/cloudflare/skills/cloudflare/references/email-routing/configuration.md b/plugins/cloudflare/skills/cloudflare/references/email-routing/configuration.md new file mode 100644 index 0000000..3f9613e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-routing/configuration.md @@ -0,0 +1,186 @@ +# Email Routing Configuration + +## Wrangler Configuration + +### Basic Email Worker + +```jsonc +// wrangler.jsonc +{ + "name": "email-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", + "send_email": [{ "name": "EMAIL" }] +} +``` + +```typescript +// src/index.ts +export default { + async email(message, env, ctx) { + await message.forward("destination@example.com"); + } +} satisfies ExportedHandler; +``` + +### With Storage Bindings + +```jsonc +{ + "name": "email-processor", + "send_email": [{ "name": "EMAIL" }], + "kv_namespaces": [{ "binding": "KV", "id": "abc123" }], + "r2_buckets": [{ "binding": "R2", "bucket_name": "emails" }], + "d1_databases": [{ "binding": "DB", "database_id": "def456" }] +} +``` + +```typescript +interface Env { + EMAIL: SendEmail; + KV: KVNamespace; + R2: R2Bucket; + DB: D1Database; +} +``` + +## Local Development + +```bash +npx wrangler dev + +# Test with curl +curl -X POST 'http://localhost:8787/__email' \ + --header 'content-type: message/rfc822' \ + --data 'From: test@example.com +To: you@yourdomain.com +Subject: Test + +Body' +``` + +## Deployment + +```bash +npx wrangler deploy +``` + +**Connect to Email Routing:** + +Dashboard: Email > Email Routing > [domain] > Settings > Email Workers > Select worker + +API: +```bash +curl -X PUT "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/email/routing/settings" \ + -H "Authorization: Bearer $API_TOKEN" \ + -d '{"enabled": true, "worker": "email-worker"}' +``` + +## DNS (Auto-Created) + +```dns +yourdomain.com. IN MX 1 isaac.mx.cloudflare.net. +yourdomain.com. IN MX 2 linda.mx.cloudflare.net. +yourdomain.com. IN MX 3 amir.mx.cloudflare.net. +yourdomain.com. IN TXT "v=spf1 include:_spf.mx.cloudflare.net ~all" +``` + +## Secrets & Variables + +```bash +# Secrets (encrypted) +npx wrangler secret put API_KEY + +# Variables (plain) +# wrangler.jsonc +{ "vars": { "THRESHOLD": "5.0" } } +``` + +```typescript +interface Env { + API_KEY: string; + THRESHOLD: string; +} +``` + +## TypeScript Setup + +```bash +npm install --save-dev @cloudflare/workers-types +``` + +```json +// tsconfig.json +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022"], + "types": ["@cloudflare/workers-types"], + "moduleResolution": "bundler", + "strict": true + } +} +``` + +```typescript +import type { ForwardableEmailMessage } from "@cloudflare/workers-types"; + +export default { + async email(message: ForwardableEmailMessage, env: Env, ctx: ExecutionContext): Promise { + await message.forward("dest@example.com"); + } +} satisfies ExportedHandler; +``` + +## Dependencies + +```bash +npm install postal-mime +``` + +```typescript +import PostalMime from 'postal-mime'; + +export default { + async email(message, env, ctx) { + const parser = new PostalMime(); + const email = await parser.parse(await message.raw.arrayBuffer()); + console.log(email.subject); + await message.forward("inbox@corp.com"); + } +} satisfies ExportedHandler; +``` + +## Multi-Environment + +```bash +# wrangler.dev.jsonc +{ "name": "worker-dev", "vars": { "ENV": "dev" } } + +# wrangler.prod.jsonc +{ "name": "worker-prod", "vars": { "ENV": "prod" } } + +npx wrangler deploy --config wrangler.dev.jsonc +npx wrangler deploy --config wrangler.prod.jsonc +``` + +## CI/CD (GitHub Actions) + +```yaml +# .github/workflows/deploy.yml +name: Deploy +on: + push: + branches: [main] +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + - run: npm ci + - run: npx wrangler deploy + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/email-routing/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/email-routing/gotchas.md new file mode 100644 index 0000000..20ea419 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-routing/gotchas.md @@ -0,0 +1,196 @@ +# Gotchas & Troubleshooting + +## Critical Pitfalls + +### Stream Consumption (MOST COMMON) + +**Problem:** "stream already consumed" or worker hangs + +**Cause:** `message.raw` is `ReadableStream` - consume once only + +**Solution:** +```typescript +// ❌ WRONG +const email1 = await parser.parse(await message.raw.arrayBuffer()); +const email2 = await parser.parse(await message.raw.arrayBuffer()); // FAILS + +// ✅ CORRECT +const raw = await message.raw.arrayBuffer(); +const email = await parser.parse(raw); +``` + +Consume `message.raw` immediately before any async operations. + +### Destination Verification + +**Problem:** Emails not forwarding + +**Cause:** Destination unverified + +**Solution:** Add destination, check inbox for verification email, click link. Verify status: `GET /zones/{id}/email/routing/addresses` + +### Mail Authentication + +**Problem:** Legitimate emails rejected + +**Cause:** Missing SPF/DKIM/DMARC on sender domain + +**Solution:** Configure sender DNS: +```dns +example.com. IN TXT "v=spf1 include:_spf.example.com ~all" +selector._domainkey.example.com. IN TXT "v=DKIM1; k=rsa; p=..." +_dmarc.example.com. IN TXT "v=DMARC1; p=quarantine" +``` + +### Envelope vs Header + +**Problem:** Filtering on wrong address + +**Solution:** +```typescript +// Routing/auth: envelope +if (message.from === "trusted@example.com") { } + +// Display: headers +const display = message.headers.get("from"); +``` + +### SendEmail Limits + +| Issue | Limit | Solution | +|-------|-------|----------| +| From domain | Must own | Use Email Routing domain | +| Volume | ~100/min Free | Upgrade or throttle | +| Attachments | Not supported | Link to R2 | +| Type | Transactional | No bulk | + +## Common Errors + +### CPU Time Exceeded + +**Cause:** Heavy parsing, large emails + +**Solution:** +```typescript +const size = parseInt(message.headers.get("content-length") || "0") / 1024 / 1024; +if (size > 20) { + message.setReject("Too large"); + return; +} + +ctx.waitUntil(expensiveWork()); +await message.forward("dest@example.com"); +``` + +### Rule Not Triggering + +**Causes:** Priority conflict, matcher error, catch-all override + +**Solution:** Check priority (lower=first), verify exact match, confirm destination verified + +### Undefined Property + +**Cause:** Missing header + +**Solution:** +```typescript +// ❌ WRONG +const subj = message.headers.get("subject").toLowerCase(); + +// ✅ CORRECT +const subj = message.headers.get("subject")?.toLowerCase() || ""; +``` + +## Limits + +| Resource | Free | Paid | +|----------|------|------| +| Email size | 25 MB | 25 MB | +| Rules | 200 | 200 | +| Destinations | 200 | 200 | +| CPU time | 10ms | 50ms | +| SendEmail | ~100/min | Higher | + +## Debugging + +### Local + +```bash +npx wrangler dev + +curl -X POST 'http://localhost:8787/__email' \ + --header 'content-type: message/rfc822' \ + --data 'From: test@example.com +To: you@yourdomain.com +Subject: Test + +Body' +``` + +### Production + +```bash +npx wrangler tail +``` + +### Pattern + +```typescript +export default { + async email(message, env, ctx) { + try { + console.log("From:", message.from); + await process(message, env); + } catch (err) { + console.error(err); + message.setReject(err.message); + } + } +} satisfies ExportedHandler; +``` + +## Auth Troubleshooting + +### Check Status + +```typescript +const auth = message.headers.get("authentication-results") || ""; +console.log({ + spf: auth.includes("spf=pass"), + dkim: auth.includes("dkim=pass"), + dmarc: auth.includes("dmarc=pass") +}); + +if (!auth.includes("pass")) { + message.setReject("Failed auth"); + return; +} +``` + +### SPF Issues + +**Causes:** Forwarding breaks SPF, too many lookups (>10), missing includes + +**Solution:** +```dns +; ✅ Good +example.com. IN TXT "v=spf1 include:_spf.google.com ~all" + +; ❌ Bad - too many +example.com. IN TXT "v=spf1 include:a.com include:b.com ... ~all" +``` + +### DMARC Alignment + +**Cause:** From domain must match SPF/DKIM domain + +## Best Practices + +1. Consume `message.raw` immediately +2. Verify destinations +3. Handle missing headers (`?.`) +4. Use envelope for routing +5. Check spam scores +6. Test locally first +7. Use `ctx.waitUntil` for background work +8. Size-check early diff --git a/plugins/cloudflare/skills/cloudflare/references/email-routing/patterns.md b/plugins/cloudflare/skills/cloudflare/references/email-routing/patterns.md new file mode 100644 index 0000000..2163677 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-routing/patterns.md @@ -0,0 +1,229 @@ +# Common Patterns + +## 1. Allowlist/Blocklist + +```typescript +// Allowlist +const allowed = ["user@example.com", "trusted@corp.com"]; +if (!allowed.includes(message.from)) { + message.setReject("Not allowed"); + return; +} +await message.forward("inbox@corp.com"); +``` + +## 2. Parse Email Body + +```typescript +import PostalMime from 'postal-mime'; + +export default { + async email(message, env, ctx) { + // CRITICAL: Consume stream immediately + const raw = await message.raw.arrayBuffer(); + + const parser = new PostalMime(); + const email = await parser.parse(raw); + + console.log({ + subject: email.subject, + text: email.text, + html: email.html, + from: email.from.address, + attachments: email.attachments.length + }); + + await message.forward("inbox@corp.com"); + } +} satisfies ExportedHandler; +``` + +## 3. Spam Filter + +```typescript +const score = parseFloat(message.headers.get("x-cf-spamh-score") || "0"); +if (score > 5) { + message.setReject("Spam detected"); + return; +} +await message.forward("inbox@corp.com"); +``` + +## 4. Archive to R2 + +```typescript +interface Env { R2: R2Bucket; } + +export default { + async email(message, env, ctx) { + const raw = await message.raw.arrayBuffer(); + + const key = `${new Date().toISOString()}-${message.from}.eml`; + await env.R2.put(key, raw, { + httpMetadata: { contentType: "message/rfc822" } + }); + + await message.forward("inbox@corp.com"); + } +} satisfies ExportedHandler; +``` + +## 5. Store Metadata in KV + +```typescript +import PostalMime from 'postal-mime'; + +interface Env { KV: KVNamespace; } + +export default { + async email(message, env, ctx) { + const raw = await message.raw.arrayBuffer(); + const parser = new PostalMime(); + const email = await parser.parse(raw); + + const metadata = { + from: email.from.address, + subject: email.subject, + timestamp: new Date().toISOString(), + size: raw.byteLength + }; + + await env.KV.put(`email:${Date.now()}`, JSON.stringify(metadata)); + await message.forward("inbox@corp.com"); + } +} satisfies ExportedHandler; +``` + +## 6. Subject-Based Routing + +```typescript +export default { + async email(message, env, ctx) { + const subject = message.headers.get("subject")?.toLowerCase() || ""; + + if (subject.includes("[urgent]")) { + await message.forward("oncall@corp.com"); + } else if (subject.includes("[billing]")) { + await message.forward("billing@corp.com"); + } else if (subject.includes("[support]")) { + await message.forward("support@corp.com"); + } else { + await message.forward("general@corp.com"); + } + } +} satisfies ExportedHandler; +``` + +## 7. Auto-Reply + +```typescript +interface Env { + EMAIL: SendEmail; + REPLIED: KVNamespace; +} + +export default { + async email(message, env, ctx) { + const msgId = message.headers.get("message-id"); + + if (msgId && await env.REPLIED.get(msgId)) { + await message.forward("archive@corp.com"); + return; + } + + ctx.waitUntil((async () => { + await env.EMAIL.send({ + from: "noreply@yourdomain.com", + to: message.from, + subject: "Re: " + (message.headers.get("subject") || ""), + text: "Thank you. We'll respond within 24h." + }); + if (msgId) await env.REPLIED.put(msgId, "1", { expirationTtl: 604800 }); + })()); + + await message.forward("support@corp.com"); + } +} satisfies ExportedHandler; +``` + +## 8. Extract Attachments + +```typescript +import PostalMime from 'postal-mime'; + +interface Env { ATTACHMENTS: R2Bucket; } + +export default { + async email(message, env, ctx) { + const parser = new PostalMime(); + const email = await parser.parse(await message.raw.arrayBuffer()); + + for (const att of email.attachments) { + const key = `${Date.now()}-${att.filename}`; + await env.ATTACHMENTS.put(key, att.content, { + httpMetadata: { contentType: att.mimeType } + }); + } + + await message.forward("inbox@corp.com"); + } +} satisfies ExportedHandler; +``` + +## 9. Log to D1 + +```typescript +import PostalMime from 'postal-mime'; + +interface Env { DB: D1Database; } + +export default { + async email(message, env, ctx) { + const parser = new PostalMime(); + const email = await parser.parse(await message.raw.arrayBuffer()); + + ctx.waitUntil( + env.DB.prepare("INSERT INTO log (ts, from_addr, subj) VALUES (?, ?, ?)") + .bind(new Date().toISOString(), email.from.address, email.subject || "") + .run() + ); + + await message.forward("inbox@corp.com"); + } +} satisfies ExportedHandler; +``` + +## 10. Multi-Tenant + +```typescript +interface Env { TENANTS: KVNamespace; } + +export default { + async email(message, env, ctx) { + const subdomain = message.to.split("@")[1].split(".")[0]; + const config = await env.TENANTS.get(subdomain, "json") as { forward: string } | null; + + if (!config) { + message.setReject("Unknown tenant"); + return; + } + + await message.forward(config.forward); + } +} satisfies ExportedHandler; +``` + +## Summary + +| Pattern | Use Case | Storage | +|---------|----------|---------| +| Allowlist | Security | None | +| Parse | Body/attachments | None | +| Spam Filter | Reduce spam | None | +| R2 Archive | Email storage | R2 | +| KV Meta | Analytics | KV | +| Subject Route | Dept routing | None | +| Auto-Reply | Support | KV | +| Attachments | Doc mgmt | R2 | +| D1 Log | Audit trail | D1 | +| Multi-Tenant | SaaS | KV | diff --git a/plugins/cloudflare/skills/cloudflare/references/email-workers/README.md b/plugins/cloudflare/skills/cloudflare/references/email-workers/README.md new file mode 100644 index 0000000..5a3e304 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-workers/README.md @@ -0,0 +1,151 @@ +# Cloudflare Email Workers + +Process incoming emails programmatically using Cloudflare Workers runtime. + +## Overview + +Email Workers enable custom email processing logic at the edge. Build spam filters, auto-responders, ticket systems, notification handlers, and more using the same Workers runtime you use for HTTP requests. + +**Key capabilities**: +- Process inbound emails with full message access +- Forward to verified destinations +- Send replies with proper threading +- Parse MIME content and attachments +- Integrate with KV, R2, D1, and external APIs + +## Quick Start + +### Minimal ES Modules Handler + +```typescript +export default { + async email(message, env, ctx) { + // Reject spam + if (message.from.includes('spam.com')) { + message.setReject('Blocked'); + return; + } + + // Forward to inbox + await message.forward('inbox@example.com'); + } +}; +``` + +### Core Operations + +| Operation | Method | Use Case | +|-----------|--------|----------| +| Forward | `message.forward(to, headers?)` | Route to verified destination | +| Reject | `message.setReject(reason)` | Block with SMTP error | +| Reply | `message.reply(emailMessage)` | Auto-respond with threading | +| Parse | postal-mime library | Extract subject, body, attachments | + +## Reading Order + +For comprehensive understanding, read files in this order: + +1. **README.md** (this file) - Overview and quick start +2. **configuration.md** - Setup, deployment, bindings +3. **api.md** - Complete API reference +4. **patterns.md** - Real-world implementation examples +5. **gotchas.md** - Critical pitfalls and debugging + +## In This Reference + +| File | Description | Key Topics | +|------|-------------|------------| +| [api.md](./api.md) | Complete API reference | ForwardableEmailMessage, SendEmail bindings, reply() method, postal-mime/mimetext APIs | +| [configuration.md](./configuration.md) | Setup and configuration | wrangler.jsonc, bindings, deployment, dependencies | +| [patterns.md](./patterns.md) | Real-world examples | Allowlists from KV, auto-reply with threading, attachment extraction, webhook notifications | +| [gotchas.md](./gotchas.md) | Pitfalls and debugging | Stream consumption, ctx.waitUntil errors, security, limits | + +## Architecture + +``` +Incoming Email → Email Routing → Email Worker + ↓ + Process + Decide + ↓ + ┌───────────────┼───────────────┐ + ↓ ↓ ↓ + Forward Reply Reject +``` + +**Event flow**: +1. Email arrives at your domain +2. Email Routing matches route (e.g., `support@example.com`) +3. Bound Email Worker receives `ForwardableEmailMessage` +4. Worker processes and takes action (forward/reply/reject) +5. Email delivered or rejected based on worker logic + +## Key Concepts + +### Envelope vs Headers + +- **Envelope addresses** (`message.from`, `message.to`): SMTP transport addresses (trusted) +- **Header addresses** (parsed from body): Display addresses (can be spoofed) + +Use envelope addresses for security decisions. + +### Single-Use Streams + +`message.raw` is a ReadableStream that can only be read once. Buffer to ArrayBuffer for multiple uses. + +```typescript +// Buffer first +const buffer = await new Response(message.raw).arrayBuffer(); +const email = await PostalMime.parse(buffer); +``` + +See [gotchas.md](./gotchas.md#readablestream-can-only-be-consumed-once) for details. + +### Verified Destinations + +`forward()` only works with addresses verified in the Cloudflare Email Routing dashboard. Add destinations before deployment. + +## Use Cases + +- **Spam filtering**: Block based on sender, content, or reputation +- **Auto-responders**: Send acknowledgment replies with threading +- **Ticket creation**: Parse emails and create support tickets +- **Email archival**: Store in KV, R2, or D1 +- **Notification routing**: Forward to Slack, Discord, or webhooks +- **Attachment processing**: Extract files to R2 storage +- **Multi-tenant routing**: Route based on recipient subdomain +- **Size filtering**: Reject oversized attachments + +## Limits + +| Limit | Value | +|-------|-------| +| Max message size | 25 MiB | +| Max routing rules | 200 | +| Max destinations | 200 | +| CPU time (free tier) | 10ms | +| CPU time (paid tier) | 50ms | + +See [gotchas.md](./gotchas.md#limits-reference) for complete limits table. + +## Prerequisites + +Before deploying Email Workers: + +1. **Enable Email Routing** in Cloudflare dashboard for your domain +2. **Verify destination addresses** for forwarding +3. **Configure DMARC/SPF** for sending domains (required for replies) +4. **Set up wrangler.jsonc** with SendEmail binding + +See [configuration.md](./configuration.md) for detailed setup. + +## Service Worker Syntax (Deprecated) + +Modern projects should use ES modules format shown above. Service Worker syntax (`addEventListener('email', ...)`) is deprecated but still supported. + +## See Also + +- [Email Routing Documentation](https://developers.cloudflare.com/email-routing/) +- [Workers Platform](https://developers.cloudflare.com/workers/) +- [Wrangler CLI](https://developers.cloudflare.com/workers/wrangler/) +- [postal-mime on npm](https://www.npmjs.com/package/postal-mime) +- [mimetext on npm](https://www.npmjs.com/package/mimetext) diff --git a/plugins/cloudflare/skills/cloudflare/references/email-workers/api.md b/plugins/cloudflare/skills/cloudflare/references/email-workers/api.md new file mode 100644 index 0000000..74da66c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-workers/api.md @@ -0,0 +1,237 @@ +# Email Workers API Reference + +Complete API reference for Cloudflare Email Workers runtime. + +## ForwardableEmailMessage Interface + +The main interface passed to email handlers. + +```typescript +interface ForwardableEmailMessage { + readonly from: string; // Envelope MAIL FROM (SMTP sender) + readonly to: string; // Envelope RCPT TO (SMTP recipient) + readonly headers: Headers; // Web-standard Headers object + readonly raw: ReadableStream; // Raw MIME message (single-use stream) + readonly rawSize: number; // Total message size in bytes + + setReject(reason: string): void; + forward(rcptTo: string, headers?: Headers): Promise; + reply(message: EmailMessage): Promise; +} +``` + +### Properties + +| Property | Type | Description | +|----------|------|-------------| +| `from` | string | Envelope sender (SMTP MAIL FROM) - use for security | +| `to` | string | Envelope recipient (SMTP RCPT TO) | +| `headers` | Headers | Message headers (Subject, Message-ID, etc.) | +| `raw` | ReadableStream | Raw MIME message (**single-use**, buffer first) | +| `rawSize` | number | Message size in bytes | + +### Methods + +#### setReject(reason: string): void + +Reject with permanent SMTP 5xx error. Email not delivered, sender may receive bounce. + +```typescript +if (blockList.includes(message.from)) { + message.setReject('Sender blocked'); +} +``` + +#### forward(rcptTo: string, headers?: Headers): Promise + +Forward to verified destination. Only `X-*` custom headers allowed. + +```typescript +await message.forward('inbox@example.com'); + +// With custom headers +const h = new Headers(); +h.set('X-Processed-By', 'worker'); +await message.forward('inbox@example.com', h); +``` + +#### reply(message: EmailMessage): Promise + +Send a reply to the original sender (March 2025 feature). + +```typescript +import { EmailMessage } from 'cloudflare:email'; +import { createMimeMessage } from 'mimetext'; + +const msg = createMimeMessage(); +msg.setSender({ name: 'Support', addr: 'support@example.com' }); +msg.setRecipient(message.from); +msg.setSubject(`Re: ${message.headers.get('Subject')}`); +msg.setHeader('In-Reply-To', message.headers.get('Message-ID')); +msg.setHeader('References', message.headers.get('References') || ''); +msg.addMessage({ + contentType: 'text/plain', + data: 'Thank you for your message.' +}); + +await message.reply(new EmailMessage( + 'support@example.com', + message.from, + msg.asRaw() +)); +``` + +**Requirements**: +- Incoming email needs valid DMARC +- Reply once per event, recipient = `message.from` +- Sender domain = receiving domain, with DMARC/SPF/DKIM +- Max 100 `References` entries +- Threading: `In-Reply-To` (original Message-ID), `References`, new `Message-ID` + +## EmailMessage Constructor + +```typescript +import { EmailMessage } from 'cloudflare:email'; + +new EmailMessage(from: string, to: string, raw: ReadableStream | string) +``` + +Used for sending emails (replies or via SendEmail binding). Domain must be verified. + +## SendEmail Interface + +```typescript +interface SendEmail { + send(message: EmailMessage): Promise; +} + +// Usage +await env.EMAIL.send(new EmailMessage(from, to, mimeContent)); +``` + +## SendEmail Binding Types + +```jsonc +{ + "send_email": [ + { "name": "EMAIL" }, // Type 1: Any verified address + { "name": "LOGS", "destination_address": "logs@example.com" }, // Type 2: Single dest + { "name": "TEAM", "allowed_destination_addresses": ["a@ex.com", "b@ex.com"] }, // Type 3: Dest allowlist + { "name": "NOREPLY", "allowed_sender_addresses": ["noreply@ex.com"] } // Type 4: Sender allowlist + ] +} +``` + +## postal-mime Parsed Output + +postal-mime v2.7.3 parses incoming emails into structured data. + +```typescript +interface ParsedEmail { + headers: Array<{ key: string; value: string }>; + from: { name: string; address: string } | null; + to: Array<{ name: string; address: string }> | { name: string; address: string } | null; + cc: Array<{ name: string; address: string }> | null; + bcc: Array<{ name: string; address: string }> | null; + subject: string; + messageId: string | null; + inReplyTo: string | null; + references: string | null; + date: string | null; + html: string | null; + text: string | null; + attachments: Array<{ + filename: string; + mimeType: string; + disposition: string | null; + related: boolean; + contentId: string | null; + content: Uint8Array; + }>; +} +``` + +### Usage + +```typescript +import PostalMime from 'postal-mime'; + +const buffer = await new Response(message.raw).arrayBuffer(); +const email = await PostalMime.parse(buffer); + +console.log(email.subject); +console.log(email.from?.address); +console.log(email.text); +console.log(email.attachments.length); +``` + +## mimetext API Quick Reference + +mimetext v3.0.27 composes outgoing emails. + +```typescript +import { createMimeMessage } from 'mimetext'; + +const msg = createMimeMessage(); + +// Sender +msg.setSender({ name: 'John Doe', addr: 'john@example.com' }); + +// Recipients +msg.setRecipient('alice@example.com'); +msg.setRecipients(['bob@example.com', 'carol@example.com']); +msg.setCc('manager@example.com'); +msg.setBcc(['audit@example.com']); + +// Headers +msg.setSubject('Meeting Notes'); +msg.setHeader('In-Reply-To', ''); +msg.setHeader('References', ' '); +msg.setHeader('Message-ID', `<${crypto.randomUUID()}@example.com>`); + +// Content +msg.addMessage({ + contentType: 'text/plain', + data: 'Plain text content' +}); + +msg.addMessage({ + contentType: 'text/html', + data: '

HTML content

' +}); + +// Attachments +msg.addAttachment({ + filename: 'report.pdf', + contentType: 'application/pdf', + data: pdfBuffer // Uint8Array or base64 string +}); + +// Generate raw MIME +const raw = msg.asRaw(); // Returns string +``` + +## TypeScript Types + +```typescript +import { + ForwardableEmailMessage, + EmailMessage +} from 'cloudflare:email'; + +interface Env { + EMAIL: SendEmail; + EMAIL_ARCHIVE: KVNamespace; + ALLOWED_SENDERS: KVNamespace; +} + +export default { + async email( + message: ForwardableEmailMessage, + env: Env, + ctx: ExecutionContext + ): Promise { + // Fully typed + } +}; +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/email-workers/configuration.md b/plugins/cloudflare/skills/cloudflare/references/email-workers/configuration.md new file mode 100644 index 0000000..7928d04 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-workers/configuration.md @@ -0,0 +1,112 @@ +# Email Workers Configuration + +## wrangler.jsonc + +```jsonc +{ + "name": "email-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-27", + "send_email": [ + { "name": "EMAIL" }, // Unrestricted + { "name": "EMAIL_LOGS", "destination_address": "logs@example.com" }, // Single dest + { "name": "EMAIL_TEAM", "allowed_destination_addresses": ["a@ex.com", "b@ex.com"] }, + { "name": "EMAIL_NOREPLY", "allowed_sender_addresses": ["noreply@ex.com"] } + ], + "kv_namespaces": [{ "binding": "ARCHIVE", "id": "xxx" }], + "r2_buckets": [{ "binding": "ATTACHMENTS", "bucket_name": "email-attachments" }], + "vars": { "WEBHOOK_URL": "https://hooks.example.com" } +} +``` + +## TypeScript Types + +```typescript +interface Env { + EMAIL: SendEmail; + ARCHIVE: KVNamespace; + ATTACHMENTS: R2Bucket; + WEBHOOK_URL: string; +} + +export default { + async email(message: ForwardableEmailMessage, env: Env, ctx: ExecutionContext) {} +}; +``` + +## Dependencies + +```bash +npm install postal-mime mimetext +npm install -D @cloudflare/workers-types wrangler typescript +``` + +Use postal-mime v2.x, mimetext v3.x. + +## tsconfig.json + +```json +{ + "compilerOptions": { + "target": "ES2022", "module": "ES2022", "lib": ["ES2022"], + "types": ["@cloudflare/workers-types"], + "moduleResolution": "bundler", "strict": true + } +} +``` + +## Local Development + +```bash +npx wrangler dev + +# Test receiving +curl --request POST 'http://localhost:8787/cdn-cgi/handler/email' \ + --url-query 'from=sender@example.com' --url-query 'to=recipient@example.com' \ + --header 'Content-Type: text/plain' --data-raw 'Subject: Test\n\nHello' +``` + +Sent emails write to local `.eml` files. + +## Deployment Checklist + +- [ ] Enable Email Routing in dashboard +- [ ] Verify destination addresses +- [ ] Configure DMARC/SPF/DKIM for sending +- [ ] Create KV/R2 resources if needed +- [ ] Update wrangler.jsonc with production IDs + +```bash +npx wrangler deploy +npx wrangler deployments list +``` + +## Dashboard Setup + +1. **Email Routing:** Domain → Email → Enable Email Routing +2. **Verify addresses:** Email → Destination addresses → Add & verify +3. **Bind Worker:** Email → Email Workers → Create route → Select pattern & Worker +4. **DMARC:** Add TXT `_dmarc.domain.com`: `v=DMARC1; p=quarantine;` + +## Secrets + +```bash +npx wrangler secret put API_KEY +# Access: env.API_KEY +``` + +## Monitoring + +```bash +npx wrangler tail +npx wrangler tail --status error +npx wrangler tail --format json +``` + +## Troubleshooting + +| Error | Fix | +|-------|-----| +| "Binding not found" | Check `send_email` name matches code | +| "Invalid destination" | Verify in Email Routing dashboard | +| Type errors | Install `@cloudflare/workers-types` | diff --git a/plugins/cloudflare/skills/cloudflare/references/email-workers/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/email-workers/gotchas.md new file mode 100644 index 0000000..3700a50 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-workers/gotchas.md @@ -0,0 +1,125 @@ +# Email Workers Gotchas + +## Critical Issues + +### ReadableStream Single-Use + +```typescript +// ❌ WRONG: Stream consumed twice +const email = await PostalMime.parse(await new Response(message.raw).arrayBuffer()); +const rawText = await new Response(message.raw).text(); // EMPTY! + +// ✅ CORRECT: Buffer first +const buffer = await new Response(message.raw).arrayBuffer(); +const email = await PostalMime.parse(buffer); +const rawText = new TextDecoder().decode(buffer); +``` + +### ctx.waitUntil() Errors Silent + +```typescript +// ❌ Errors dropped silently +ctx.waitUntil(fetch(webhookUrl, { method: 'POST', body: data })); + +// ✅ Catch and log +ctx.waitUntil( + fetch(webhookUrl, { method: 'POST', body: data }) + .catch(err => env.ERROR_LOG.put(`error:${Date.now()}`, err.message)) +); +``` + +## Security + +### Envelope vs Header From (Spoofing) + +```typescript +const envelopeFrom = message.from; // SMTP MAIL FROM (trusted) +const headerFrom = (await PostalMime.parse(buffer)).from?.address; // (untrusted) +// Use envelope for security decisions +``` + +### Input Validation + +```typescript +if (message.rawSize > 5_000_000) { message.setReject('Too large'); return; } +if ((message.headers.get('Subject') || '').length > 1000) { + message.setReject('Invalid subject'); return; +} +``` + +### DMARC for Replies + +Replies fail silently without DMARC. Verify: `dig TXT _dmarc.example.com` + +## Parsing + +### Address Parsing + +```typescript +const email = await PostalMime.parse(buffer); +const fromAddress = email.from?.address || 'unknown'; +const toAddresses = Array.isArray(email.to) ? email.to.map(t => t.address) : [email.to?.address]; +``` + +### Character Encoding + +Let postal-mime handle decoding - `email.subject`, `email.text`, `email.html` are UTF-8. + +## API Behavior + +### setReject() vs throw + +```typescript +// setReject() for SMTP rejection +if (blockList.includes(message.from)) { message.setReject('Blocked'); return; } + +// throw for worker errors +if (!env.KV) throw new Error('KV not configured'); +``` + +### forward() Only X-* Headers + +```typescript +headers.set('X-Processed-By', 'worker'); // ✅ Works +headers.set('Subject', 'Modified'); // ❌ Dropped +``` + +### Reply Requires Verified Domain + +```typescript +// Use same domain as receiving address +const receivingDomain = message.to.split('@')[1]; +await message.reply(new EmailMessage(`noreply@${receivingDomain}`, message.from, rawMime)); +``` + +## Performance + +### CPU Limit + +```typescript +// Skip parsing large emails +if (message.rawSize > 5_000_000) { + await message.forward('inbox@example.com'); + return; +} +``` + +Monitor: `npx wrangler tail` + +## Limits + +| Limit | Value | +|-------|-------| +| Max message size | 25 MiB | +| Max rules/zone | 200 | +| CPU time (free/paid) | 10ms / 50ms | +| Reply References | 100 | + +## Common Errors + +| Error | Fix | +|-------|-----| +| "Address not verified" | Add in Email Routing dashboard | +| "Exceeded CPU time" | Use `ctx.waitUntil()` or upgrade | +| "Stream is locked" | Buffer `message.raw` first | +| Silent reply failure | Check DMARC records | diff --git a/plugins/cloudflare/skills/cloudflare/references/email-workers/patterns.md b/plugins/cloudflare/skills/cloudflare/references/email-workers/patterns.md new file mode 100644 index 0000000..f1e65f5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/email-workers/patterns.md @@ -0,0 +1,102 @@ +# Email Workers Patterns + +## Parse Email + +```typescript +import PostalMime from 'postal-mime'; + +export default { + async email(message, env, ctx) { + const buffer = await new Response(message.raw).arrayBuffer(); + const email = await PostalMime.parse(buffer); + console.log(email.from, email.subject, email.text, email.attachments.length); + await message.forward('inbox@example.com'); + } +}; +``` + +## Filtering + +```typescript +// Allowlist from KV +const allowList = await env.ALLOWED_SENDERS.get('list', 'json') || []; +if (!allowList.includes(message.from)) { + message.setReject('Not allowed'); + return; +} + +// Size check (avoid parsing large emails) +if (message.rawSize > 5_000_000) { + await message.forward('inbox@example.com'); // Forward without parsing + return; +} +``` + +## Auto-Reply with Threading + +```typescript +import { EmailMessage } from 'cloudflare:email'; +import { createMimeMessage } from 'mimetext'; + +const msg = createMimeMessage(); +msg.setSender({ addr: 'support@example.com' }); +msg.setRecipient(message.from); +msg.setSubject(`Re: ${message.headers.get('Subject')}`); +msg.setHeader('In-Reply-To', message.headers.get('Message-ID') || ''); +msg.addMessage({ contentType: 'text/plain', data: 'Thank you. We will respond.' }); + +await message.reply(new EmailMessage('support@example.com', message.from, msg.asRaw())); +``` + +## Rate-Limited Auto-Reply + +```typescript +const rateKey = `rate:${message.from}`; +if (!await env.RATE_LIMIT.get(rateKey)) { + // Send reply... + ctx.waitUntil(env.RATE_LIMIT.put(rateKey, '1', { expirationTtl: 3600 })); +} +``` + +## Subject-Based Routing + +```typescript +const subject = (message.headers.get('Subject') || '').toLowerCase(); +if (subject.includes('billing')) await message.forward('billing@example.com'); +else if (subject.includes('support')) await message.forward('support@example.com'); +else await message.forward('general@example.com'); +``` + +## Multi-Tenant Routing + +```typescript +// support+tenant123@example.com → tenant123 +const tenantId = message.to.split('@')[0].match(/\+(.+)$/)?.[1] || 'default'; +const config = await env.TENANT_CONFIG.get(tenantId, 'json'); +config?.forwardTo ? await message.forward(config.forwardTo) : message.setReject('Unknown'); +``` + +## Archive & Extract Attachments + +```typescript +// Archive to KV +ctx.waitUntil(env.ARCHIVE.put(`email:${Date.now()}`, JSON.stringify({ + from: message.from, subject: email.subject +}))); + +// Attachments to R2 +for (const att of email.attachments) { + ctx.waitUntil(env.R2.put(`${Date.now()}-${att.filename}`, att.content)); +} +``` + +## Webhook Integration + +```typescript +ctx.waitUntil( + fetch(env.WEBHOOK_URL, { + method: 'POST', + body: JSON.stringify({ from: message.from, subject: message.headers.get('Subject') }) + }).catch(err => console.error(err)) +); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/graphql-api/README.md b/plugins/cloudflare/skills/cloudflare/references/graphql-api/README.md new file mode 100644 index 0000000..3f92dd5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/graphql-api/README.md @@ -0,0 +1,147 @@ +# Cloudflare GraphQL Analytics API + +Query analytics data across all Cloudflare products via a single GraphQL endpoint. Covers HTTP requests, Workers metrics, DNS, Firewall events, Network Analytics, and 70+ other datasets. + +## Overview + +- **Single endpoint** for all analytics: `https://api.cloudflare.com/client/v4/graphql` +- **1,400+ schema types** spanning every Cloudflare product +- **Two scopes**: zone-level (per-domain) and account-level (cross-domain) +- **Adaptive sampling** on high-traffic datasets with confidence intervals +- **No mutations** - read-only analytics (the Mutation type is a stub) +- **Cost-based rate limiting** - default 300 queries per 5 minutes per user (max 320, varies by query cost) + +## Quick Decision Tree + +``` +Need analytics data from Cloudflare? +├─ HTTP traffic (requests, bandwidth, cache) → httpRequestsAdaptiveGroups (zone or account) +├─ Workers performance (CPU, wall time, errors) → workersInvocationsAdaptive (account) +├─ Firewall/WAF events → firewallEventsAdaptive / firewallEventsAdaptiveGroups (zone or account) +├─ DNS query analytics → dnsAnalyticsAdaptive / dnsAnalyticsAdaptiveGroups (zone or account) +├─ Network layer (DDoS, Magic Transit) → *NetworkAnalyticsAdaptiveGroups (account) +├─ Storage (R2, KV, D1, DO) → r2OperationsAdaptiveGroups / kvOperationsAdaptiveGroups / etc. (account) +├─ AI (Workers AI, AI Gateway) → aiInferenceAdaptive / aiGatewayRequestsAdaptiveGroups (account) +├─ Load Balancing → loadBalancingRequestsAdaptiveGroups (zone) +├─ Custom high-cardinality metrics → Workers Analytics Engine (see ../analytics-engine/) +└─ Need raw logs, not aggregates → Logpush (see Cloudflare docs) +``` + +## Core Concepts + +| Concept | Description | +|---------|-------------| +| **Endpoint** | `POST https://api.cloudflare.com/client/v4/graphql` | +| **Explorer** | [graphql.cloudflare.com](https://graphql.cloudflare.com/) - interactive query builder | +| **Viewer** | Root query object: `viewer { zones(...) { ... } }` or `viewer { accounts(...) { ... } }` | +| **Dataset (Node)** | A queryable table under a zone or account (e.g., `httpRequestsAdaptiveGroups`) | +| **Dimensions** | Fields to group by (time buckets, country, status code, script name, etc.) | +| **Metrics** | Aggregation fields: `count`, `sum { ... }`, `avg { ... }`, `quantiles { ... }`, `ratio { ... }` | +| **Filter** | Input object constraining results by time range, dimensions, etc. | +| **Limit** | Maximum rows returned per dataset node (required, max varies by dataset) | +| **OrderBy** | Enum-based sorting: `[field_ASC]` or `[field_DESC]` | +| **Adaptive Sampling** | Nodes with `Adaptive` in the name use ABR sampling; results are statistically representative | + +## Query Structure + +Every query follows this pattern: + +```graphql +{ + viewer { + # Zone-scoped + zones(filter: { zoneTag: "ZONE_ID" }) { + datasetName( + filter: { datetime_gt: "...", datetime_lt: "..." } + limit: 1000 + orderBy: [datetimeFiveMinutes_DESC] + ) { + count + dimensions { ... } + sum { ... } + } + } + # Account-scoped + accounts(filter: { accountTag: "ACCOUNT_ID" }) { + datasetName(filter: { ... }, limit: 100) { + count + dimensions { ... } + sum { ... } + } + } + } +} +``` + +## Dataset Naming Convention + +Dataset names follow a consistent pattern visible in the schema: + +| Pattern | Meaning | Example | +|---------|---------|---------| +| `*Adaptive` | Raw rows with adaptive sampling; some (e.g., `workersInvocationsAdaptive`) also support aggregation fields (`sum`, `quantiles`, `avg`) | `httpRequestsAdaptive`, `workersInvocationsAdaptive` | +| `*AdaptiveGroups` | Aggregated data with adaptive sampling | `httpRequestsAdaptiveGroups` | +| `*1hGroups` | Hourly rollups (pre-aggregated) | `httpRequests1hGroups` | +| `*1dGroups` | Daily rollups (pre-aggregated) | `httpRequests1dGroups` | +| `*1mGroups` | Minutely rollups | `httpRequests1mGroups` | +| `Zone*` prefix | Zone-scoped dataset | `ZoneHttpRequestsAdaptiveGroups` | +| `Account*` prefix | Account-scoped dataset | `AccountWorkersInvocationsAdaptive` | + +**Prefer `*AdaptiveGroups` nodes** for most use cases - they support flexible time grouping via dimension fields (`datetimeFiveMinutes`, `datetimeHour`, etc.) and are the most commonly used. + +## Key Datasets by Product + +### Zone-Scoped (per-domain) + +| Dataset | Description | +|---------|-------------| +| `httpRequestsAdaptiveGroups` | HTTP traffic: requests, bytes, cache status, bot scores, WAF scores | +| `httpRequests1hGroups` / `1dGroups` / `1mGroups` | Pre-aggregated HTTP rollups (hourly/daily/minutely) | +| `firewallEventsAdaptiveGroups` | WAF, rate limiting, bot management, firewall rule events | +| `dnsAnalyticsAdaptiveGroups` | DNS query volumes, response codes, query types | +| `loadBalancingRequestsAdaptiveGroups` | Load Balancer origin request metrics | +| `pageShieldReportsAdaptiveGroups` | Page Shield CSP reports | + +### Account-Scoped (cross-domain) + +| Dataset | Description | +|---------|-------------| +| `workersInvocationsAdaptive` | Workers: requests, errors, CPU time, wall time, subrequests | +| `durableObjectsInvocationsAdaptiveGroups` | DO invocations | +| `durableObjectsStorageGroups` / `durableObjectsPeriodicGroups` | DO storage and periodic metrics | +| `d1AnalyticsAdaptiveGroups` / `d1QueriesAdaptiveGroups` | D1 database analytics | +| `r2OperationsAdaptiveGroups` / `r2StorageAdaptiveGroups` | R2 operations and storage | +| `kvOperationsAdaptiveGroups` / `kvStorageAdaptiveGroups` | KV operations and storage | +| `aiInferenceAdaptiveGroups` | Workers AI inference metrics | +| `aiGatewayRequestsAdaptiveGroups` | AI Gateway request analytics | +| `pagesFunctionsInvocationsAdaptiveGroups` | Pages Functions metrics | +| `magicTransitNetworkAnalyticsAdaptiveGroups` | Magic Transit packet/byte analytics | +| `spectrumNetworkAnalyticsAdaptiveGroups` | Spectrum TCP/UDP analytics | +| `gatewayL7RequestsAdaptiveGroups` | Zero Trust Gateway HTTP metrics | +| `gatewayResolverQueriesAdaptiveGroups` | Zero Trust Gateway DNS metrics | + +## Reading Order + +| Task | Start Here | Then Read | +|------|------------|-----------| +| **First query** | [configuration.md](configuration.md) (auth) -> this README (structure) | [api.md](api.md) | +| **Build a dashboard** | [patterns.md](patterns.md) (time-series, top-N) | [api.md](api.md) (aggregation fields) | +| **Debug query issues** | [gotchas.md](gotchas.md) | [api.md](api.md) (filtering) | +| **Understand sampling** | [gotchas.md](gotchas.md) (sampling section) | [api.md](api.md) (confidence intervals) | +| **Product-specific metrics** | [patterns.md](patterns.md) (per-product examples) | [api.md](api.md) (dataset reference) | + +## In This Reference + +- **[api.md](api.md)** - Query structure, aggregation fields (sum/avg/quantiles/count), filtering operators, dimensions, dataset details +- **[configuration.md](configuration.md)** - Authentication, API tokens, client setup (curl, JS, Python), introspection +- **[patterns.md](patterns.md)** - Common queries: time-series, top-N, Workers metrics, HTTP analytics, firewall events, multi-zone +- **[gotchas.md](gotchas.md)** - Rate limits, sampling caveats, query cost, common errors, plan-based limits + +## See Also + +- [GraphQL Analytics API Docs](https://developers.cloudflare.com/analytics/graphql-api/) +- [GraphQL API Explorer](https://graphql.cloudflare.com/) +- [Observability Reference](../observability/) - Workers Logs, Tail Workers, console logging +- [Analytics Engine Reference](../analytics-engine/) - Custom high-cardinality analytics via Workers +- [Web Analytics Reference](../web-analytics/) - Client-side (RUM) analytics +- [API Reference](../api/) - REST API, SDKs, authentication basics diff --git a/plugins/cloudflare/skills/cloudflare/references/graphql-api/api.md b/plugins/cloudflare/skills/cloudflare/references/graphql-api/api.md new file mode 100644 index 0000000..15d1a33 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/graphql-api/api.md @@ -0,0 +1,175 @@ +# GraphQL Analytics API Reference + +## Query Root + +The schema has a single entry point: `Query.viewer`. Mutations are not supported. + +```graphql +{ + cost # uint64 -- query cost (returned in response) + viewer { + budget # uint64 -- remaining budget + zones(filter: { zoneTag: "..." }) { ... } + accounts(filter: { accountTag: "..." }) { ... } + } +} +``` + +## Aggregation Fields + +Aggregated dataset nodes (`*Groups`) return these field categories. Not every node has all — use introspection to check. + +### count + +Total events in the group. Available on `*Groups` nodes but **not** on raw `*Adaptive` nodes (e.g., `workersInvocationsAdaptive` — use `sum { requests }` instead). + +### sum + +Cumulative metrics. Fields vary by dataset: + +```graphql +# HTTP requests +sum { edgeResponseBytes edgeRequestBytes visits edgeTimeToFirstByteMs originResponseDurationMs } + +# Workers invocations +sum { requests errors subrequests cpuTimeUs wallTime duration responseBodySize clientDisconnects requestDuration } +``` + +### quantiles + +Percentile distributions (on datasets like `workersInvocationsAdaptive`). Available percentiles: P25, P50, P75, P90, P95, P99, P999 for `cpuTime`, `wallTime`, `requestDuration`, `duration`, `responseBodySize`. + +```graphql +quantiles { cpuTimeP50 cpuTimeP99 wallTimeP50 wallTimeP99 } +``` + +### ratio, avg, uniq, confidence + +```graphql +ratio { status4xx status5xx } # float64 (0 to 1) -- HTTP datasets only +avg { sampleInterval } # useful for understanding sampling resolution +uniq { uniques } # unique IP count -- rollup datasets (*1hGroups, *1dGroups) only +confidence(level: 0.95) { # Adaptive datasets only; works on count and sum fields + count { estimate lower upper sampleSize } +} +``` + +## Dimensions + +Dimensions are fields you can group by via the `dimensions` sub-selection. + +### Time Dimensions + +| Dimension | Granularity | +|-----------|------------| +| `date` | Day | +| `datetime` | Exact timestamp | +| `datetimeMinute` | 1 minute | +| `datetimeFiveMinutes` | 5 minutes | +| `datetimeFifteenMinutes` | 15 minutes | +| `datetimeHour` | 1 hour | + +Workers datasets also support `datetimeSixHours`. + +### HTTP Request Dimensions (httpRequestsAdaptiveGroups) + +83 dimensions available. Key ones: + +| Dimension | Description | +|-----------|-------------| +| `clientCountryName` | Country of origin | +| `clientRequestHTTPHost` | Requested hostname | +| `clientRequestHTTPMethodName` | HTTP method | +| `clientRequestPath` | URI path | +| `edgeResponseStatus` | Edge HTTP status code | +| `cacheStatus` | Cache status (hit, miss, dynamic, etc.) | +| `coloCode` | Cloudflare datacenter IATA code | +| `clientIP` / `clientAsn` | Client IP address / ASN | +| `botScore` / `botManagementDecision` | Bot management score (0-99) / verdict | +| `wafAttackScore` / `securityAction` | WAF score / firewall action taken | +| `ja3Hash` / `ja4` | TLS fingerprints | +| `sampleInterval` | ABR sample interval | + +### Workers Dimensions (workersInvocationsAdaptive) + +`scriptName`, `scriptTag`, `scriptVersion`, `environmentName`, `status`, `usageModel`, `coloCode`, `dispatchNamespaceName`, `isDispatcher` + +### Firewall Dimensions (firewallEventsAdaptive) + +`action`, `source`, `ruleId`, `clientCountryName`, `clientIP`, `clientAsn`, `userAgent` + +## Filtering + +### Scope Filters + +```graphql +zones(filter: { zoneTag: "ZONE_ID" }) # up to 10 zones +zones(filter: { zoneTag_in: ["Z1", "Z2"] }) +accounts(filter: { accountTag: "ACCOUNT_ID" }) # exactly 1 account +``` + +### Dataset Filters + +**Always include a time range filter.** Multiple filters at the same level are implicitly AND-ed. + +```graphql +httpRequestsAdaptiveGroups( + filter: { datetime_gt: "2025-01-01T00:00:00Z", datetime_lt: "2025-01-02T00:00:00Z", clientCountryName: "US" } + limit: 1000 +) +``` + +### Filter Operators + +| Operator | Meaning | Example | +|----------|---------|---------| +| (none) | equals | `clientCountryName: "US"` | +| `_gt` / `_lt` | greater / less than | `datetime_gt: "..."` | +| `_geq` / `_leq` | greater/less or equal | `datetime_geq: "..."` | +| `_neq` | not equal | `cacheStatus_neq: "hit"` | +| `_in` / `_notin` | in / not in list | `clientCountryName_in: ["US", "GB"]` | +| `_like` / `_notlike` | SQL LIKE with `%` | `clientRequestPath_like: "/api/%"` | +| `_has` / `_hasall` / `_hasany` | array contains | `botDetectionIds_has: "abc"` | + +> `_notin` and `_notlike` are in the schema but not in official docs. Confirmed via introspection. + +### Boolean Operators (AND / OR) + +```graphql +# Explicit AND +filter: { AND: [{ datetime_gt: "..." }, { datetime_lt: "..." }, { clientCountryName: "US" }] } + +# Explicit OR +filter: { datetime_gt: "...", OR: [{ edgeResponseStatus: 403 }, { edgeResponseStatus: 429 }] } +``` + +## Pagination & Sorting + +No cursor-based pagination. Use `limit`, `orderBy`, and filter-based offsets: + +```graphql +# First page +httpRequestsAdaptiveGroups(filter: { datetime_gt: "..." }, limit: 100, orderBy: [datetime_ASC]) + +# Next page: filter by last seen value from previous page +httpRequestsAdaptiveGroups(filter: { datetime_gt: "2025-01-01T01:35:00Z" }, limit: 100, orderBy: [datetime_ASC]) +``` + +Sort with `orderBy: [field_ASC]` or `[field_DESC]`. Multiple sort fields supported. + +## Settings Node + +Query per-node limits and availability: + +```graphql +viewer { zones(filter: { zoneTag: "..." }) { settings { + httpRequestsAdaptiveGroups { enabled maxDuration maxNumberOfFields maxPageSize notOlderThan } +} } } +``` + +## See Also + +- [README.md](README.md) - Overview, decision tree, dataset index +- [configuration.md](configuration.md) - Authentication, client setup, introspection queries +- [patterns.md](patterns.md) - Common query patterns (time-series, top-N, per-product) +- [gotchas.md](gotchas.md) - Rate limits, sampling, troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/graphql-api/configuration.md b/plugins/cloudflare/skills/cloudflare/references/graphql-api/configuration.md new file mode 100644 index 0000000..d85d1a6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/graphql-api/configuration.md @@ -0,0 +1,118 @@ +# GraphQL Analytics API Configuration + +## Authentication + +### API Token (Recommended) + +| Permission | Scope | Use Case | +|------------|-------|----------| +| **Account Analytics: Read** | Account-wide | Workers, R2, KV, D1, DO, AI, Network Analytics | +| **Zone Analytics: Read** | Per-zone | HTTP requests, Firewall, DNS, Load Balancing | +| **All zones - Analytics: Read** | All zones | Multi-zone HTTP/Firewall/DNS queries | + +Create tokens at: [dash.cloudflare.com > Account API Tokens](https://dash.cloudflare.com/?to=/:account/api-tokens) + +```bash +# Verify token +curl -s https://api.cloudflare.com/client/v4/graphql \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"query":"{ viewer { zones(filter: {zoneTag: \"ZONE_ID\"}) { httpRequestsAdaptiveGroups(limit: 1, filter: {datetime_gt: \"2025-01-01T00:00:00Z\"}) { count } } } }"}' +``` + +### API Key + Email (Legacy) + +Not recommended. Use `X-Auth-Email` + `X-Auth-Key` headers instead of `Authorization: Bearer`. + +## Client Setup + +### curl + +```bash +curl -s https://api.cloudflare.com/client/v4/graphql \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{ + "query": "query($zoneTag: string!, $start: Time!, $end: Time!) { viewer { zones(filter: {zoneTag: $zoneTag}) { httpRequestsAdaptiveGroups(filter: {datetime_gt: $start, datetime_lt: $end}, limit: 10, orderBy: [datetimeFiveMinutes_DESC]) { count dimensions { datetimeFiveMinutes } } } } }", + "variables": { "zoneTag": "ZONE_ID", "start": "2025-01-01T00:00:00Z", "end": "2025-01-02T00:00:00Z" } + }' | jq . +``` + +### TypeScript / JavaScript + +```typescript +const GRAPHQL_ENDPOINT = "https://api.cloudflare.com/client/v4/graphql"; + +async function queryGraphQL(query: string, variables: Record = {}): Promise { + const response = await fetch(GRAPHQL_ENDPOINT, { + method: "POST", + headers: { Authorization: `Bearer ${process.env.CF_API_TOKEN}`, "Content-Type": "application/json" }, + body: JSON.stringify({ query, variables }), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`); + const json = await response.json() as { data: T | null; errors?: { message: string }[] }; + if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join("; ")); + return json.data!; +} +``` + +### Python + +```python +import requests, os + +def query_graphql(query: str, variables: dict = None) -> dict: + r = requests.post("https://api.cloudflare.com/client/v4/graphql", + headers={"Authorization": f"Bearer {os.environ['CF_API_TOKEN']}", "Content-Type": "application/json"}, + json={"query": query, "variables": variables or {}}) + r.raise_for_status() + result = r.json() + if result.get("errors"): + raise Exception("; ".join(e["message"] for e in result["errors"])) + return result["data"] +``` + +### From a Cloudflare Worker + +Store the API token as a secret (`CF_API_TOKEN`). Use standard `fetch` to POST to `https://api.cloudflare.com/client/v4/graphql` with the same JSON body format as above. Always check `response.errors` — GraphQL returns 200 even on query failures. + +## GraphQL API Explorer + +Interactive explorer at [graphql.cloudflare.com](https://graphql.cloudflare.com/) — provides schema docs, autocomplete, variable panel, and shareable queries. Authenticates via your Cloudflare dashboard session. + +## Schema Introspection + +```graphql +# List zone-scoped datasets +{ __type(name: "zone") { fields { name description } } } + +# List account-scoped datasets +{ __type(name: "account") { fields { name description } } } + +# Discover dimensions for a dataset +{ __type(name: "ZoneHttpRequestsAdaptiveGroupsDimensions") { + fields { name type { name kind } } +} } + +# Discover filter operators for a dataset +{ __type(name: "ZoneHttpRequestsAdaptiveGroupsFilter_InputObject") { + inputFields { name type { name kind } } +} } +``` + +## Finding Your Zone and Account IDs + +- **Zone ID**: Dashboard > select zone > Overview (right sidebar), or via API +- **Account ID**: Dashboard > Account Home URL, or via API + +```bash +curl -s https://api.cloudflare.com/client/v4/zones -H "Authorization: Bearer $CF_API_TOKEN" | jq '.result[] | {name, id}' +curl -s https://api.cloudflare.com/client/v4/accounts -H "Authorization: Bearer $CF_API_TOKEN" | jq '.result[] | {name, id}' +``` + +## See Also + +- [README.md](README.md) - Overview, decision tree, dataset index +- [api.md](api.md) - Query structure, aggregation fields, filtering operators +- [patterns.md](patterns.md) - Common query patterns (time-series, top-N, per-product) +- [gotchas.md](gotchas.md) - Rate limits, sampling, troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/graphql-api/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/graphql-api/gotchas.md new file mode 100644 index 0000000..18df7c6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/graphql-api/gotchas.md @@ -0,0 +1,110 @@ +# GraphQL Analytics API Gotchas & Troubleshooting + +## Rate Limits + +| Limit | Value | +|-------|-------| +| GraphQL queries per user | **Default 300 per 5 minutes** (max 320, at least 1/sec) | +| General API rate limit | 1200 requests per 5 minutes (shared across all API calls) | +| Zone scope per query | Up to **10 zones** | +| Account scope per query | Exactly **1 account** | + +The GraphQL rate limit is separate from the general API limit. Exceeding either results in `HTTP 429` and blocks all API calls for 5 minutes. Enterprise customers can contact support to raise limits. + +### "429 Too Many Requests" + +**Cause:** Exceeded rate limit. + +**Solution:** Batch multiple datasets into single queries, cache results, increase intervals between queries. Use `{ viewer { budget } }` to monitor remaining budget. + +## Sampling & Data Accuracy + +### Adaptive Bit Rate (ABR) Sampling + +Datasets with `Adaptive` in the name use adaptive sampling: +- Results are **statistically representative**, not exact +- Same query may return **slightly different numbers** each run +- Higher traffic = higher sampling rate = more accurate +- `sampleInterval` dimension shows the ratio (1 = no sampling, 10 = ~1-in-10 sampled) + +For high-confidence numbers, use `confidence(level: 0.95)` to get estimate bounds. For exact counts, use rollup nodes (`httpRequests1hGroups`, `httpRequests1dGroups`) which are pre-aggregated without sampling. + +### Rollup vs. Adaptive + +| Feature | Rollup (`*1hGroups`, `*1dGroups`) | Adaptive (`*AdaptiveGroups`) | +|---------|-----------------------------------|-----------------------------| +| Sampling | No (pre-aggregated) | Yes (ABR) | +| Flexibility | Fixed time buckets | Any granularity | +| Dimensions | Fewer | Many more | +| Accuracy | Exact | Statistical estimate | + +## Common Errors + +### "Access denied" / "authentication error" + +**Cause:** Token lacks required permission or wrong scope. + +**Solution:** Account-scoped queries need **Account Analytics: Read**. Zone-scoped queries need **Zone Analytics: Read**. Verify: `curl -s https://api.cloudflare.com/client/v4/user/tokens/verify -H "Authorization: Bearer $TOKEN"` + +### "field not found" / "Cannot query field" + +**Cause:** Wrong dataset name, nonexistent field, or wrong scope (zone vs. account). + +**Solution:** Names are case-sensitive camelCase (`httpRequestsAdaptiveGroups`). Zone datasets go under `zones(...)`, account datasets under `accounts(...)`. Use introspection to verify. + +### "filter is required" / empty results + +**Cause:** Missing required time range filter or incorrect zone/account tag. + +**Solution:** Always include `datetime_gt` / `datetime_lt` (or `_geq` / `_leq`). + +### "limit is required" / "limit exceeds maximum" + +**Cause:** Missing `limit` or exceeding node's max page size. + +**Solution:** Always specify `limit`. Max varies by dataset (typically 10,000 for groups, 100 for raw events). Check via settings query. + +### "query is too complex" / "query exceeds budget" + +**Cause:** Too many fields, datasets, or too broad a time range. + +**Solution:** Reduce time range, request fewer dimensions/metrics, break into smaller queries. Monitor `cost` and `budget` in responses. + +### 200 Response with Errors + +GraphQL returns HTTP 200 even on failures. **Always check `response.errors`:** + +```json +{ "data": null, "errors": [{ "message": "filter is required for httpRequestsAdaptiveGroups" }] } +``` + +## Plan-Based Availability + +Not all datasets are available on all plans. Higher plans get more datasets, longer retention (`notOlderThan`), wider time ranges (`maxDuration`), more fields, and larger page sizes. + +### "node is not available" / "node is disabled" + +**Cause:** Dataset not on your plan, or product not enabled. + +**Solution:** Check `settings { { enabled } }`. Some datasets require specific subscriptions (e.g., Network Analytics requires Magic Transit/Spectrum). + +## DateTime & Timezone Handling + +- All times are **UTC only** (ISO 8601: `"2025-01-15T10:30:00Z"`) +- `Date` type: `"2025-01-15"` (used in `date_geq`/`date_leq` for storage datasets) +- `Time` type: `"2025-01-15T10:30:00Z"` (used in `datetime_gt`/`datetime_lt`) +- Filters are start-inclusive: events that start within the window are included + +## Performance Tips + +- **Narrow time ranges** are faster and cheaper +- **Select only needed dimensions** — each additional dimension increases cost +- **Use rollup nodes** (`*1dGroups`) for simple daily totals without dimension breakdowns +- **Batch datasets** into one query instead of separate HTTP requests + +## See Also + +- [README.md](README.md) - Overview, decision tree, dataset index +- [api.md](api.md) - Query structure, aggregation fields, filtering operators +- [configuration.md](configuration.md) - Authentication, client setup, introspection queries +- [patterns.md](patterns.md) - Common query patterns (time-series, top-N, per-product) diff --git a/plugins/cloudflare/skills/cloudflare/references/graphql-api/patterns.md b/plugins/cloudflare/skills/cloudflare/references/graphql-api/patterns.md new file mode 100644 index 0000000..f02c36b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/graphql-api/patterns.md @@ -0,0 +1,225 @@ +# GraphQL Analytics API Patterns & Best Practices + +## Time-Series Queries + +Use time dimension granularity matching your range (see Best Practices below). + +```graphql +query TrafficTimeSeries($zoneTag: string!, $start: Time!, $end: Time!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + httpRequestsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end } + limit: 1000 + orderBy: [datetimeFiveMinutes_ASC] # or datetimeHour_ASC for longer ranges + ) { + count + dimensions { datetimeFiveMinutes } + sum { edgeResponseBytes } + ratio { status4xx status5xx } + } + } + } +} +``` + +## Top-N Queries + +### Top Countries by Request Count + +```graphql +query TopCountries($zoneTag: string!, $start: Time!, $end: Time!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + httpRequestsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end } + limit: 10 + orderBy: [count_DESC] + ) { + count + dimensions { clientCountryName } + } + } + } +} +``` + +Use `orderBy: [sum_edgeResponseBytes_DESC]` for top paths by bandwidth. Add `edgeResponseStatus_geq: 400` to the filter for top error status codes. + +## Workers Analytics + +```graphql +query WorkersOverview($accountTag: string!, $start: Time!, $end: Time!) { + viewer { + accounts(filter: { accountTag: $accountTag }) { + workersInvocationsAdaptive( + filter: { datetime_gt: $start, datetime_lt: $end } + limit: 100 + orderBy: [sum_requests_DESC] + ) { + sum { requests errors subrequests wallTime } + quantiles { cpuTimeP50 cpuTimeP99 wallTimeP50 wallTimeP99 } + dimensions { scriptName } + } + } + } +} +``` + +Filter by `scriptName` for a specific Worker. Add `datetimeFiveMinutes` dimension + `orderBy: [datetimeFiveMinutes_ASC]` for error rate over time. + +## Firewall / Security + +```graphql +query RecentFirewallEvents($zoneTag: string!, $start: Time!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + firewallEventsAdaptive( + filter: { datetime_gt: $start } + limit: 50 + orderBy: [datetime_DESC] + ) { + action source clientIP clientCountryName userAgent + clientRequestHTTPHost clientRequestPath ruleId datetime + } + } + } +} +``` + +For aggregated firewall stats, use `firewallEventsAdaptiveGroups` with `action: "block"` filter and group by `ruleId`, `source`, `datetimeHour`. + +## DNS Analytics + +```graphql +query DNSQueryVolume($zoneTag: string!, $start: Time!, $end: Time!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + dnsAnalyticsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end } + limit: 500 + orderBy: [datetimeFiveMinutes_ASC] + ) { + count + dimensions { datetimeFiveMinutes } + } + } + } +} +``` + +## Storage Analytics (Account-Scoped) + +R2, KV, and D1 use `date` (Date type) filters instead of `datetime` (Time type). + +```graphql +# R2 operations +r2OperationsAdaptiveGroups(filter: { date_geq: $start, date_leq: $end }, limit: 100, orderBy: [date_DESC]) { + dimensions { date bucketName actionType } + sum { requests } +} + +# KV operations +kvOperationsAdaptiveGroups(filter: { date_geq: $start, date_leq: $end }, limit: 100, orderBy: [date_DESC]) { + dimensions { date actionType } + sum { requests } +} + +# D1 analytics +d1AnalyticsAdaptiveGroups(filter: { date_geq: $start, date_leq: $end }, limit: 100, orderBy: [date_DESC]) { + dimensions { date databaseId } + sum { readQueries writeQueries rowsRead rowsWritten } +} +``` + +## Cache Analytics + +```graphql +query CacheStatusBreakdown($zoneTag: string!, $start: Time!, $end: Time!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + httpRequestsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end } + limit: 20 + orderBy: [count_DESC] + ) { + count + dimensions { cacheStatus } + sum { edgeResponseBytes } + } + } + } +} +``` + +For cache hit ratio over time, use aliases to query the same dataset twice — once with `cacheStatus: "hit"` filter and once without — then compute the ratio client-side. + +## Multi-Dataset Queries + +A single request can query multiple datasets, avoiding extra HTTP round-trips: + +```graphql +query DashboardOverview($zoneTag: string!, $start: Time!, $end: Time!) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + httpTraffic: httpRequestsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end }, limit: 1 + ) { count sum { edgeResponseBytes } ratio { status4xx status5xx } } + firewallEvents: firewallEventsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end }, limit: 5, orderBy: [count_DESC] + ) { count dimensions { action source } } + dnsQueries: dnsAnalyticsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end }, limit: 1 + ) { count } + } + } +} +``` + +## AI & Gateway Analytics + +```graphql +# Workers AI inference +aiInferenceAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end }, limit: 100, orderBy: [datetimeHour_DESC] +) { + count + sum { totalInputTokens totalOutputTokens totalRequestBytesIn } + dimensions { modelId datetimeHour } +} + +# AI Gateway requests +aiGatewayRequestsAdaptiveGroups( + filter: { datetime_gt: $start, datetime_lt: $end }, limit: 100, orderBy: [datetimeHour_DESC] +) { + count + dimensions { gateway provider model datetimeHour } + sum { cachedTokensIn cachedTokensOut uncachedTokensIn uncachedTokensOut } +} +``` + +Both are account-scoped — nest under `accounts(filter: { accountTag: $accountTag })`. + +## Best Practices + +**Always include time filters.** Queries without time filters scan all data and are slow/expensive. + +**Match time granularity to range:** + +| Time Range | Recommended Dimension | +|------------|----------------------| +| < 6 hours | `datetimeMinute` or `datetimeFiveMinutes` | +| 6-48 hours | `datetimeFiveMinutes` or `datetimeFifteenMinutes` | +| 2-14 days | `datetimeHour` | +| 14+ days | `date` | + +**Use aliases** for querying the same dataset with different filters in one request. + +**Request only needed fields.** Extra dimensions and metrics increase query cost. + +## See Also + +- [README.md](README.md) - Overview, decision tree, dataset index +- [api.md](api.md) - Query structure, aggregation fields, filtering operators +- [configuration.md](configuration.md) - Authentication, client setup, introspection queries +- [gotchas.md](gotchas.md) - Rate limits, sampling, troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/hyperdrive/README.md b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/README.md new file mode 100644 index 0000000..6626776 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/README.md @@ -0,0 +1,82 @@ +# Hyperdrive + +Accelerates database queries from Workers via connection pooling, edge setup, query caching. + +## Key Features + +- **Connection Pooling**: Persistent connections eliminate TCP/TLS/auth handshakes (~7 round-trips) +- **Edge Setup**: Connection negotiation at edge, pooling near origin +- **Query Caching**: Auto-cache non-mutating queries (default 60s TTL) +- **Support**: PostgreSQL, MySQL + compatibles (CockroachDB, Timescale, PlanetScale, Neon, Supabase) + +## Architecture + +``` +Worker → Edge (setup) → Pool (near DB) → Origin + ↓ cached reads + Cache +``` + +## Quick Start + +```bash +# Create config +npx wrangler hyperdrive create my-db \ + --connection-string="postgres://user:pass@host:5432/db" + +# wrangler.jsonc +{ + "compatibility_flags": ["nodejs_compat"], + "hyperdrive": [{"binding": "HYPERDRIVE", "id": ""}] +} +``` + +```typescript +import { Client } from "pg"; + +export default { + async fetch(req: Request, env: Env): Promise { + const client = new Client({ + connectionString: env.HYPERDRIVE.connectionString, + }); + await client.connect(); + const result = await client.query("SELECT * FROM users WHERE id = $1", [123]); + await client.end(); + return Response.json(result.rows); + }, +}; +``` + +## When to Use + +✅ Global access to single-region DBs, high read ratios, popular queries, connection-heavy loads +❌ Write-heavy, real-time data (<1s), single-region apps close to DB + +**💡 Pair with Smart Placement** for Workers making multiple queries - executes near DB to minimize latency. + +## Driver Choice + +| Driver | Use When | Notes | +|--------|----------|-------| +| **pg** (recommended) | General use, TypeScript, ecosystem compatibility | Stable, widely used, works with most ORMs | +| **postgres.js** | Advanced features, template literals, streaming | Lighter than pg, `prepare: true` is default | +| **mysql2** | MySQL/MariaDB/PlanetScale | MySQL only, less mature support | + +## Reading Order + +| New to Hyperdrive | Implementing | Troubleshooting | +|-------------------|--------------|-----------------| +| 1. README (this) | 1. [configuration.md](./configuration.md) | 1. [gotchas.md](./gotchas.md) | +| 2. [configuration.md](./configuration.md) | 2. [api.md](./api.md) | 2. [patterns.md](./patterns.md) | +| 3. [api.md](./api.md) | 3. [patterns.md](./patterns.md) | 3. [api.md](./api.md) | + +## In This Reference +- [configuration.md](./configuration.md) - Setup, wrangler config, Smart Placement +- [api.md](./api.md) - Binding APIs, query patterns, driver usage +- [patterns.md](./patterns.md) - Use cases, ORMs, multi-query optimization +- [gotchas.md](./gotchas.md) - Limits, troubleshooting, connection management + +## See Also +- [smart-placement](../smart-placement/) - Optimize multi-query Workers near databases +- [d1](../d1/) - Serverless SQLite alternative for edge-native apps +- [workers](../workers/) - Worker runtime with database bindings diff --git a/plugins/cloudflare/skills/cloudflare/references/hyperdrive/api.md b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/api.md new file mode 100644 index 0000000..0e587b9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/api.md @@ -0,0 +1,143 @@ +# API Reference + +See [README.md](./README.md) for overview, [configuration.md](./configuration.md) for setup. + +## Binding Interface + +```typescript +interface Hyperdrive { + connectionString: string; // PostgreSQL + // MySQL properties: + host: string; + port: number; + user: string; + password: string; + database: string; +} + +interface Env { + HYPERDRIVE: Hyperdrive; +} +``` + +**Generate types:** `npx wrangler types` (auto-creates worker-configuration.d.ts from wrangler.jsonc) + +## PostgreSQL (node-postgres) - RECOMMENDED + +```typescript +import { Client } from "pg"; // pg@^8.17.2 + +export default { + async fetch(req: Request, env: Env): Promise { + const client = new Client({connectionString: env.HYPERDRIVE.connectionString}); + try { + await client.connect(); + const result = await client.query("SELECT * FROM users WHERE id = $1", [123]); + return Response.json(result.rows); + } finally { + await client.end(); + } + }, +}; +``` + +**⚠️ Workers connection limit: 6 per Worker invocation** - use connection pooling wisely. + +## PostgreSQL (postgres.js) + +```typescript +import postgres from "postgres"; // postgres@^3.4.8 + +const sql = postgres(env.HYPERDRIVE.connectionString, { + max: 5, // Limit per Worker (Workers max: 6) + prepare: true, // Enabled by default, required for caching + fetch_types: false, // Reduce latency if not using arrays +}); + +const users = await sql`SELECT * FROM users WHERE active = ${true} LIMIT 10`; +``` + +**⚠️ `prepare: true` is enabled by default and required for Hyperdrive caching.** Setting to `false` disables prepared statements + cache. + +## MySQL (mysql2) + +```typescript +import { createConnection } from "mysql2/promise"; // mysql2@^3.16.2 + +const conn = await createConnection({ + host: env.HYPERDRIVE.host, + user: env.HYPERDRIVE.user, + password: env.HYPERDRIVE.password, + database: env.HYPERDRIVE.database, + port: env.HYPERDRIVE.port, + disableEval: true, // ⚠️ REQUIRED for Workers +}); + +const [results] = await conn.query("SELECT * FROM users WHERE active = ? LIMIT ?", [true, 10]); +ctx.waitUntil(conn.end()); +``` + +**⚠️ MySQL support is less mature than PostgreSQL** - expect fewer optimizations and potential edge cases. + +## Query Caching + +**Cacheable:** +```sql +SELECT * FROM posts WHERE published = true; +SELECT COUNT(*) FROM users; +``` + +**NOT cacheable:** +```sql +-- Writes +INSERT/UPDATE/DELETE + +-- Volatile functions +SELECT NOW(); +SELECT random(); +SELECT LASTVAL(); -- PostgreSQL +SELECT UUID(); -- MySQL +``` + +**Cache config:** +- Default: `max_age=60s`, `swr=15s` +- Max `max_age`: 3600s +- Disable: `--caching-disabled=true` + +**Multiple configs pattern:** +```typescript +// Reads: cached +const sqlCached = postgres(env.HYPERDRIVE_CACHED.connectionString); +const posts = await sqlCached`SELECT * FROM posts ORDER BY views DESC LIMIT 10`; + +// Writes/time-sensitive: no cache +const sqlNoCache = postgres(env.HYPERDRIVE_NO_CACHE.connectionString); +const orders = await sqlNoCache`SELECT * FROM orders WHERE created_at > NOW() - INTERVAL 5 MINUTE`; +``` + +## ORMs + +**Drizzle:** +```typescript +import { drizzle } from "drizzle-orm/postgres-js"; // drizzle-orm@^0.45.1 +import postgres from "postgres"; + +const client = postgres(env.HYPERDRIVE.connectionString, {max: 5, prepare: true}); +const db = drizzle(client); +const users = await db.select().from(users).where(eq(users.active, true)).limit(10); +``` + +**Kysely:** +```typescript +import { Kysely, PostgresDialect } from "kysely"; // kysely@^0.27+ +import postgres from "postgres"; + +const db = new Kysely({ + dialect: new PostgresDialect({ + postgres: postgres(env.HYPERDRIVE.connectionString, {max: 5, prepare: true}), + }), +}); +const users = await db.selectFrom("users").selectAll().where("active", "=", true).execute(); +``` + +See [patterns.md](./patterns.md) for use cases, [gotchas.md](./gotchas.md) for limits. diff --git a/plugins/cloudflare/skills/cloudflare/references/hyperdrive/configuration.md b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/configuration.md new file mode 100644 index 0000000..6d429a9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/configuration.md @@ -0,0 +1,159 @@ +# Configuration + +See [README.md](./README.md) for overview. + +## Create Config + +**PostgreSQL:** +```bash +# Basic +npx wrangler hyperdrive create my-db \ + --connection-string="postgres://user:pass@host:5432/db" + +# Custom cache +npx wrangler hyperdrive create my-db \ + --connection-string="postgres://..." \ + --max-age=120 --swr=30 + +# No cache +npx wrangler hyperdrive create my-db \ + --connection-string="postgres://..." \ + --caching-disabled=true +``` + +**MySQL:** +```bash +npx wrangler hyperdrive create my-db \ + --connection-string="mysql://user:pass@host:3306/db" +``` + +## wrangler.jsonc + +```jsonc +{ + "compatibility_date": "2025-01-01", // Use latest for new projects + "compatibility_flags": ["nodejs_compat"], + "hyperdrive": [ + { + "binding": "HYPERDRIVE", + "id": "", + "localConnectionString": "postgres://user:pass@localhost:5432/dev" + } + ] +} +``` + +**Generate TypeScript types:** Run `npx wrangler types` to auto-generate `worker-configuration.d.ts` from your wrangler.jsonc. + +**Multiple configs:** +```jsonc +{ + "hyperdrive": [ + {"binding": "HYPERDRIVE_CACHED", "id": ""}, + {"binding": "HYPERDRIVE_NO_CACHE", "id": ""} + ] +} +``` + +## Management + +```bash +npx wrangler hyperdrive list +npx wrangler hyperdrive get +npx wrangler hyperdrive update --max-age=180 +npx wrangler hyperdrive delete +``` + +## Config Options + +Hyperdrive create/update CLI flags: + +| Option | Default | Notes | +|--------|---------|-------| +| `--caching-disabled` | `false` | Disable caching | +| `--max-age` | `60` | Cache TTL (max 3600s) | +| `--swr` | `15` | Stale-while-revalidate | +| `--origin-connection-limit` | 20/100 | Free/paid | +| `--access-client-id` | - | Tunnel auth | +| `--access-client-secret` | - | Tunnel auth | +| `--sslmode` | `require` | PostgreSQL only | + +## Smart Placement Integration + +For Workers making **multiple queries** per request, enable Smart Placement to execute near your database: + +```jsonc +{ + "compatibility_date": "2025-01-01", + "compatibility_flags": ["nodejs_compat"], + "placement": { + "mode": "smart" + }, + "hyperdrive": [ + { + "binding": "HYPERDRIVE", + "id": "" + } + ] +} +``` + +**Benefits:** Multi-query Workers run closer to DB, reducing round-trip latency. See [patterns.md](./patterns.md) for examples. + +## Private DB via Tunnel + +``` +Worker → Hyperdrive → Access → Tunnel → Private Network → DB +``` + +**Setup:** +```bash +# 1. Create tunnel +cloudflared tunnel create my-db-tunnel + +# 2. Configure hostname in Zero Trust dashboard +# Domain: db-tunnel.example.com +# Service: TCP -> localhost:5432 + +# 3. Create service token (Zero Trust > Service Auth) +# Save Client ID/Secret + +# 4. Create Access app (db-tunnel.example.com) +# Policy: Service Auth token from step 3 + +# 5. Create Hyperdrive +npx wrangler hyperdrive create my-private-db \ + --host=db-tunnel.example.com \ + --user=dbuser --password=dbpass --database=prod \ + --access-client-id= --access-client-secret= +``` + +**⚠️ Don't specify `--port` with Tunnel** - port configured in tunnel service settings. + +## Local Dev + +**Option 1: Local (RECOMMENDED):** +```bash +# Env var (takes precedence) +export CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE="postgres://user:pass@localhost:5432/dev" +npx wrangler dev + +# wrangler.jsonc +{"hyperdrive": [{"binding": "HYPERDRIVE", "localConnectionString": "postgres://..."}]} +``` + +**Remote DB locally:** +```bash +# PostgreSQL +export CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE="postgres://user:pass@remote:5432/db?sslmode=require" + +# MySQL +export CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_HYPERDRIVE="mysql://user:pass@remote:3306/db?sslMode=REQUIRED" +``` + +**Option 2: Remote execution:** +```bash +npx wrangler dev --remote # Uses deployed config, affects production +``` + +See [api.md](./api.md), [patterns.md](./patterns.md), [gotchas.md](./gotchas.md). diff --git a/plugins/cloudflare/skills/cloudflare/references/hyperdrive/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/gotchas.md new file mode 100644 index 0000000..efa2ead --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/gotchas.md @@ -0,0 +1,77 @@ +# Gotchas + +See [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md), [patterns.md](./patterns.md). + +## Common Errors + +### "Too many open connections" / "Connection limit exceeded" + +**Cause:** Workers have a hard limit of **6 concurrent connections per invocation** +**Solution:** Set `max: 5` in driver config, reuse connections, ensure proper cleanup with `client.end()` or `ctx.waitUntil(conn.end())` + +### "Failed to acquire a connection (Pool exhausted)" + +**Cause:** All connections in pool are in use, often due to long-running transactions +**Solution:** Reduce transaction duration, avoid queries >60s, don't hold connections during external calls, or upgrade to paid plan for more connections + +### "connection_refused" + +**Cause:** Database refusing connections due to firewall, connection limits, or service down +**Solution:** Check firewall allows Cloudflare IPs, verify DB listening on port, confirm service running, and validate credentials + +### "Query timeout (deadline exceeded)" + +**Cause:** Query execution exceeding 60s timeout limit +**Solution:** Optimize with indexes, reduce dataset with LIMIT, break into smaller queries, or use async processing + +### "password authentication failed" + +**Cause:** Invalid credentials in Hyperdrive configuration +**Solution:** Check username and password in Hyperdrive config match database credentials + +### "SSL/TLS connection error" + +**Cause:** SSL/TLS configuration mismatch between Hyperdrive and database +**Solution:** Add `sslmode=require` (Postgres) or `sslMode=REQUIRED` (MySQL), upload CA cert if self-signed, verify DB has SSL enabled, and check cert expiry + +### "Queries not being cached" + +**Cause:** Query is mutating (INSERT/UPDATE/DELETE), contains volatile functions (NOW(), RANDOM()), or caching disabled +**Solution:** Verify query is non-mutating SELECT, avoid volatile functions, confirm caching enabled, use `wrangler dev --remote` to test, and set `prepare=true` for postgres.js + +### "Slow multi-query Workers despite Hyperdrive" + +**Cause:** Worker executing at edge, each query round-trips to DB region +**Solution:** Enable Smart Placement (`"placement": {"mode": "smart"}` in wrangler.jsonc) to execute Worker near DB. See [patterns.md](./patterns.md) Multi-Query pattern. + +### "Local database connection failed" + +**Cause:** `localConnectionString` incorrect or database not running +**Solution:** Verify `localConnectionString` correct, check DB running, confirm env var name matches binding, and test with psql/mysql client + +### "Environment variable not working" + +**Cause:** Environment variable format incorrect or not exported +**Solution:** Use format `CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_`, ensure binding matches wrangler.jsonc, export variable in shell, and restart wrangler dev + +## Limits + +| Limit | Free | Paid | Notes | +|-------|------|------|-------| +| Max configs | 10 | 25 | Hyperdrive configurations per account | +| Worker connections | 6 | 6 | Max concurrent connections per Worker invocation | +| Username/DB name | 63 bytes | 63 bytes | Maximum length | +| Connection timeout | 15s | 15s | Time to establish connection | +| Idle timeout | 10 min | 10 min | Connection idle timeout | +| Max origin connections | ~20 | ~100 | Connections to origin database | +| Query duration max | 60s | 60s | Queries >60s terminated | +| Cached response max | 50 MB | 50 MB | Responses >50MB returned but not cached | + +## Resources + +- [Docs](https://developers.cloudflare.com/hyperdrive/) +- [Getting Started](https://developers.cloudflare.com/hyperdrive/get-started/) +- [Wrangler Reference](https://developers.cloudflare.com/hyperdrive/reference/wrangler-commands/) +- [Supported DBs](https://developers.cloudflare.com/hyperdrive/reference/supported-databases-and-features/) +- [Discord #hyperdrive](https://discord.cloudflare.com) +- [Limit Increase Form](https://forms.gle/ukpeZVLWLnKeixDu7) diff --git a/plugins/cloudflare/skills/cloudflare/references/hyperdrive/patterns.md b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/patterns.md new file mode 100644 index 0000000..bd794b9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/hyperdrive/patterns.md @@ -0,0 +1,190 @@ +# Patterns + +See [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md). + +## High-Traffic Read-Heavy + +```typescript +const sql = postgres(env.HYPERDRIVE.connectionString, {max: 5, prepare: true}); + +// Cacheable: popular content +const posts = await sql`SELECT * FROM posts WHERE published = true ORDER BY views DESC LIMIT 20`; + +// Cacheable: user profiles +const [user] = await sql`SELECT id, username, bio FROM users WHERE id = ${userId}`; +``` + +**Benefits:** Trending/profiles cached (60s), connection pooling handles spikes. + +## Mixed Read/Write + +```typescript +interface Env { + HYPERDRIVE_CACHED: Hyperdrive; // max_age=120 + HYPERDRIVE_REALTIME: Hyperdrive; // caching disabled +} + +// Reads: cached +if (req.method === "GET") { + const sql = postgres(env.HYPERDRIVE_CACHED.connectionString, {prepare: true}); + const products = await sql`SELECT * FROM products WHERE category = ${cat}`; +} + +// Writes: no cache (immediate consistency) +if (req.method === "POST") { + const sql = postgres(env.HYPERDRIVE_REALTIME.connectionString, {prepare: true}); + await sql`INSERT INTO orders ${sql(data)}`; +} +``` + +## Analytics Dashboard + +```typescript +const client = new Client({connectionString: env.HYPERDRIVE.connectionString}); +await client.connect(); + +// Aggregate queries cached (use fixed timestamps for caching) +const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toISOString(); +const dailyStats = await client.query(` + SELECT DATE(created_at) as date, COUNT(*) as orders, SUM(amount) as revenue + FROM orders WHERE created_at >= $1 + GROUP BY DATE(created_at) ORDER BY date DESC +`, [thirtyDaysAgo]); + +const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(); +const topProducts = await client.query(` + SELECT p.name, COUNT(oi.id) as count, SUM(oi.quantity * oi.price) as revenue + FROM order_items oi JOIN products p ON oi.product_id = p.id + WHERE oi.created_at >= $1 + GROUP BY p.id, p.name ORDER BY revenue DESC LIMIT 10 +`, [sevenDaysAgo]); +``` + +**Benefits:** Expensive aggregations cached (avoid NOW() for cacheability), dashboard instant, reduced DB load. + +## Multi-Tenant + +```typescript +const tenantId = req.headers.get("X-Tenant-ID"); +const sql = postgres(env.HYPERDRIVE.connectionString, {prepare: true}); + +// Tenant-scoped queries cached separately +const docs = await sql` + SELECT * FROM documents + WHERE tenant_id = ${tenantId} AND deleted_at IS NULL + ORDER BY updated_at DESC LIMIT 50 +`; +``` + +**Benefits:** Per-tenant caching, shared connection pool, protects DB from multi-tenant load. + +## Geographically Distributed + +```typescript +// Worker runs at edge nearest user +// Connection setup at edge (fast), pooling near DB (efficient) +const sql = postgres(env.HYPERDRIVE.connectionString, {prepare: true}); +const [user] = await sql`SELECT * FROM users WHERE id = ${userId}`; + +return Response.json({ + user, + serverRegion: req.cf?.colo, // Edge location +}); +``` + +**Benefits:** Edge setup + DB pooling = global → single-region DB without replication. + +## Multi-Query + Smart Placement + +For Workers making **multiple queries** per request, enable Smart Placement to execute near DB: + +```jsonc +// wrangler.jsonc +{ + "placement": {"mode": "smart"}, + "hyperdrive": [{"binding": "HYPERDRIVE", "id": ""}] +} +``` + +```typescript +const sql = postgres(env.HYPERDRIVE.connectionString, {prepare: true}); + +// Multiple queries benefit from Smart Placement +const [user] = await sql`SELECT * FROM users WHERE id = ${userId}`; +const orders = await sql`SELECT * FROM orders WHERE user_id = ${userId} ORDER BY created_at DESC LIMIT 10`; +const stats = await sql`SELECT COUNT(*) as total, SUM(amount) as spent FROM orders WHERE user_id = ${userId}`; + +return Response.json({user, orders, stats}); +``` + +**Benefits:** Worker executes near DB → reduces latency for each query. Without Smart Placement, each query round-trips from edge. + +## Connection Pooling + +Operates in **transaction mode**: connection acquired per transaction, `RESET` on return. + +**SET statements:** +```typescript +// ✅ Within transaction +await client.query("BEGIN"); +await client.query("SET work_mem = '256MB'"); +await client.query("SELECT * FROM large_table"); // Uses SET +await client.query("COMMIT"); // RESET after + +// ✅ Single statement +await client.query("SET work_mem = '256MB'; SELECT * FROM large_table"); + +// ❌ Across queries (may get different connection) +await client.query("SET work_mem = '256MB'"); +await client.query("SELECT * FROM large_table"); // SET not applied +``` + +**Best practices:** +```typescript +// ❌ Long transactions block pooling +await client.query("BEGIN"); +await processThousands(); // Connection held entire time +await client.query("COMMIT"); + +// ✅ Short transactions +await client.query("BEGIN"); +await client.query("UPDATE users SET status = $1 WHERE id = $2", [status, id]); +await client.query("COMMIT"); + +// ✅ SET LOCAL within transaction +await client.query("BEGIN"); +await client.query("SET LOCAL work_mem = '256MB'"); +await client.query("SELECT * FROM large_table"); +await client.query("COMMIT"); +``` + +## Performance Tips + +**Enable prepared statements (required for caching):** +```typescript +const sql = postgres(connectionString, {prepare: true}); // Default, enables caching +``` + +**Optimize connection settings:** +```typescript +const sql = postgres(connectionString, { + max: 5, // Stay under Workers' 6 connection limit + fetch_types: false, // Reduce latency if not using arrays + idle_timeout: 60, // Match Worker lifetime +}); +``` + +**Write cache-friendly queries:** +```typescript +// ✅ Cacheable (deterministic) +await sql`SELECT * FROM products WHERE category = 'electronics' LIMIT 10`; + +// ❌ Not cacheable (volatile NOW()) +await sql`SELECT * FROM logs WHERE created_at > NOW()`; + +// ✅ Cacheable (parameterized timestamp) +const ts = Date.now(); +await sql`SELECT * FROM logs WHERE created_at > ${ts}`; +``` + +See [gotchas.md](./gotchas.md) for limits, troubleshooting. diff --git a/plugins/cloudflare/skills/cloudflare/references/images/README.md b/plugins/cloudflare/skills/cloudflare/references/images/README.md new file mode 100644 index 0000000..f1dd644 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/images/README.md @@ -0,0 +1,61 @@ +# Cloudflare Images Skill Reference + +**Cloudflare Images** is an end-to-end image management solution providing storage, transformation, optimization, and delivery at scale via Cloudflare's global network. + +## Quick Decision Tree + +**Need to:** +- **Transform in Worker?** → [api.md](api.md#workers-binding-api-2026-primary-method) (Workers Binding API) +- **Upload from Worker?** → [api.md](api.md#upload-from-worker) (REST API) +- **Upload from client?** → [patterns.md](patterns.md#upload-from-client-direct-creator-upload) (Direct Creator Upload) +- **Set up variants?** → [configuration.md](configuration.md#variants-configuration) +- **Serve responsive images?** → [patterns.md](patterns.md#responsive-images) +- **Add watermarks?** → [patterns.md](patterns.md#watermarking) +- **Fix errors?** → [gotchas.md](gotchas.md#common-errors) + +## Reading Order + +**For building image upload/transform feature:** +1. [configuration.md](configuration.md) - Setup Workers binding +2. [api.md](api.md#workers-binding-api-2026-primary-method) - Learn transform API +3. [patterns.md](patterns.md#upload-from-client-direct-creator-upload) - Direct upload pattern +4. [gotchas.md](gotchas.md) - Check limits and errors + +**For URL-based transforms:** +1. [configuration.md](configuration.md#variants-configuration) - Create variants +2. [api.md](api.md#url-transform-api) - URL syntax +3. [patterns.md](patterns.md#responsive-images) - Responsive patterns + +**For troubleshooting:** +1. [gotchas.md](gotchas.md#common-errors) - Error messages +2. [gotchas.md](gotchas.md#limits) - Size/format limits + +## Core Methods + +| Method | Use Case | Location | +|--------|----------|----------| +| `env.IMAGES.input().transform()` | Transform in Worker | [api.md:11](api.md) | +| REST API `/images/v1` | Upload images | [api.md:57](api.md) | +| Direct Creator Upload | Client-side upload | [api.md:127](api.md) | +| URL transforms | Static image delivery | [api.md:112](api.md) | + +## In This Reference + +- **[api.md](api.md)** - Complete API: Workers binding, REST endpoints, URL transforms +- **[configuration.md](configuration.md)** - Setup: wrangler.toml, variants, auth, signed URLs +- **[patterns.md](patterns.md)** - Patterns: responsive images, watermarks, format negotiation, caching +- **[gotchas.md](gotchas.md)** - Troubleshooting: limits, errors, best practices + +## Key Features + +- **Automatic Optimization** - AVIF/WebP format negotiation +- **On-the-fly Transforms** - Resize, crop, blur, sharpen via URL or API +- **Workers Binding** - Transform images in Workers (2026 primary method) +- **Direct Upload** - Secure client-side uploads without backend proxy +- **Global Delivery** - Cached at 300+ Cloudflare data centers +- **Watermarking** - Overlay images programmatically + +## See Also + +- [Official Docs](https://developers.cloudflare.com/images/) +- [Workers Examples](https://developers.cloudflare.com/images/tutorials/) diff --git a/plugins/cloudflare/skills/cloudflare/references/images/api.md b/plugins/cloudflare/skills/cloudflare/references/images/api.md new file mode 100644 index 0000000..c172e22 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/images/api.md @@ -0,0 +1,96 @@ +# API Reference + +## Workers Binding API + +```toml +# wrangler.toml +[images] +binding = "IMAGES" +``` + +### Transform Images + +```typescript +const imageResponse = await env.IMAGES + .input(fileBuffer) + .transform({ width: 800, height: 600, fit: "cover", quality: 85, format: "avif" }) + .output(); +return imageResponse.response(); +``` + +### Transform Options + +```typescript +interface TransformOptions { + width?: number; height?: number; + fit?: "scale-down" | "contain" | "cover" | "crop" | "pad"; + quality?: number; // 1-100 + format?: "avif" | "webp" | "jpeg" | "png"; + dpr?: number; // 1-3 + gravity?: "auto" | "left" | "right" | "top" | "bottom" | "face" | string; + sharpen?: number; // 0-10 + blur?: number; // 1-250 + rotate?: 90 | 180 | 270; + background?: string; // CSS color for pad + metadata?: "none" | "copyright" | "keep"; + brightness?: number; contrast?: number; gamma?: number; // 0-2 +} +``` + +### Draw/Watermark + +```typescript +await env.IMAGES.input(baseImage) + .draw(env.IMAGES.input(watermark).transform({ width: 100 }), { top: 10, left: 10, opacity: 0.8 }) + .output(); +``` + +## REST API + +### Upload Image + +```bash +curl -X POST https://api.cloudflare.com/client/v4/accounts/{account_id}/images/v1 \ + -H "Authorization: Bearer {token}" -F file=@image.jpg -F metadata='{"key":"value"}' +``` + +### Other Operations + +```bash +GET /accounts/{account_id}/images/v1/{image_id} # Get details +DELETE /accounts/{account_id}/images/v1/{image_id} # Delete +GET /accounts/{account_id}/images/v1?page=1 # List +``` + +## URL Transform API + +``` +https://imagedelivery.net/{hash}/{id}/width=800,height=600,fit=cover,format=avif +``` + +**Params:** `w=`, `h=`, `fit=`, `q=`, `f=`, `dpr=`, `gravity=`, `sharpen=`, `blur=`, `rotate=`, `background=`, `metadata=` + +## Direct Creator Upload + +```typescript +// 1. Get upload URL (backend) +const { result } = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/images/v2/direct_upload`, + { method: 'POST', headers: { 'Authorization': `Bearer ${token}` }, + body: JSON.stringify({ requireSignedURLs: false }) } +).then(r => r.json()); + +// 2. Client uploads to result.uploadURL +const formData = new FormData(); +formData.append('file', file); +await fetch(result.uploadURL, { method: 'POST', body: formData }); +``` + +## Error Codes + +| Code | Message | Solution | +|------|---------|----------| +| 5400 | Invalid format | Use JPEG, PNG, GIF, WebP | +| 5401 | Too large | Max 100MB | +| 5403 | Invalid transform | Check params | +| 9413 | Rate limit | Implement backoff | diff --git a/plugins/cloudflare/skills/cloudflare/references/images/configuration.md b/plugins/cloudflare/skills/cloudflare/references/images/configuration.md new file mode 100644 index 0000000..9fa2deb --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/images/configuration.md @@ -0,0 +1,211 @@ +# Configuration + +## Wrangler Integration + +### Workers Binding Setup + +Add to `wrangler.toml`: + +```toml +name = "my-image-worker" +main = "src/index.ts" +compatibility_date = "2024-01-01" + +[images] +binding = "IMAGES" +``` + +Access in Worker: + +```typescript +interface Env { + IMAGES: ImageBinding; +} + +export default { + async fetch(request: Request, env: Env): Promise { + return await env.IMAGES + .input(imageBuffer) + .transform({ width: 800 }) + .output() + .response(); + } +}; +``` + +### Upload via Script + +Wrangler doesn't have built-in Images commands, use REST API: + +```typescript +// scripts/upload-image.ts +import fs from 'fs'; +import FormData from 'form-data'; + +async function uploadImage(filePath: string) { + const accountId = process.env.CLOUDFLARE_ACCOUNT_ID!; + const apiToken = process.env.CLOUDFLARE_API_TOKEN!; + + const formData = new FormData(); + formData.append('file', fs.createReadStream(filePath)); + + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/images/v1`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiToken}`, + }, + body: formData, + } + ); + + const result = await response.json(); + console.log('Uploaded:', result); +} + +uploadImage('./photo.jpg'); +``` + +### Environment Variables + +Store account hash for URL construction: + +```toml +[vars] +IMAGES_ACCOUNT_HASH = "your-account-hash" +ACCOUNT_ID = "your-account-id" +``` + +Access in Worker: + +```typescript +const imageUrl = `https://imagedelivery.net/${env.IMAGES_ACCOUNT_HASH}/${imageId}/public`; +``` + +## Variants Configuration + +Variants are named presets for transformations. + +### Create Variant (Dashboard) + +1. Navigate to Images → Variants +2. Click "Create Variant" +3. Set name (e.g., `thumbnail`) +4. Configure: `width=200,height=200,fit=cover` + +### Create Variant (API) + +```bash +curl -X POST \ + https://api.cloudflare.com/client/v4/accounts/{account_id}/images/v1/variants \ + -H "Authorization: Bearer {api_token}" \ + -H "Content-Type: application/json" \ + -d '{ + "id": "thumbnail", + "options": { + "width": 200, + "height": 200, + "fit": "cover" + }, + "neverRequireSignedURLs": true + }' +``` + +### Use Variant + +``` +https://imagedelivery.net/{account_hash}/{image_id}/thumbnail +``` + +### Common Variant Presets + +```json +{ + "thumbnail": { + "width": 200, + "height": 200, + "fit": "cover" + }, + "avatar": { + "width": 128, + "height": 128, + "fit": "cover", + "gravity": "face" + }, + "hero": { + "width": 1920, + "height": 1080, + "fit": "cover", + "quality": 90 + }, + "mobile": { + "width": 640, + "fit": "scale-down", + "quality": 80, + "format": "avif" + } +} +``` + +## Authentication + +### API Token (Recommended) + +Generate at: Dashboard → My Profile → API Tokens + +Required permissions: +- Account → Cloudflare Images → Edit + +```bash +curl -H "Authorization: Bearer {api_token}" \ + https://api.cloudflare.com/client/v4/accounts/{account_id}/images/v1 +``` + +### API Key (Legacy) + +```bash +curl -H "X-Auth-Email: {email}" \ + -H "X-Auth-Key: {api_key}" \ + https://api.cloudflare.com/client/v4/accounts/{account_id}/images/v1 +``` + +## Signed URLs + +For private images, enable signed URLs: + +```bash +# Upload with signed URLs required +curl -X POST \ + https://api.cloudflare.com/client/v4/accounts/{account_id}/images/v1 \ + -H "Authorization: Bearer {api_token}" \ + -F file=@private.jpg \ + -F requireSignedURLs=true +``` + +Generate signed URL: + +```typescript +import { createHmac } from 'crypto'; + +function signUrl(imageId: string, variant: string, expiry: number, key: string): string { + const path = `/${imageId}/${variant}`; + const toSign = `${path}${expiry}`; + const signature = createHmac('sha256', key) + .update(toSign) + .digest('hex'); + + return `https://imagedelivery.net/{hash}${path}?exp=${expiry}&sig=${signature}`; +} + +// Sign URL valid for 1 hour +const signedUrl = signUrl('image-id', 'public', Date.now() + 3600, env.SIGNING_KEY); +``` + +## Local Development + +```bash +npx wrangler dev --remote +``` + +Must use `--remote` for Images binding access. diff --git a/plugins/cloudflare/skills/cloudflare/references/images/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/images/gotchas.md new file mode 100644 index 0000000..6f52455 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/images/gotchas.md @@ -0,0 +1,99 @@ +# Gotchas & Best Practices + +## Fit Modes + +| Mode | Best For | Behavior | +|------|----------|----------| +| `cover` | Hero images, thumbnails | Fills space, crops excess | +| `contain` | Product images, artwork | Preserves full image, may add padding | +| `scale-down` | User uploads | Never enlarges | +| `crop` | Precise crops | Uses gravity | +| `pad` | Fixed aspect ratio | Adds background | + +## Format Selection + +```typescript +format: 'auto' // Recommended - negotiates best format +``` + +**Support:** AVIF (Chrome 85+, Firefox 93+, Safari 16.4+), WebP (Chrome 23+, Firefox 65+, Safari 14+) + +## Quality Settings + +| Use Case | Quality | +|----------|---------| +| Thumbnails | 75-80 | +| Standard | 85 (default) | +| High-quality | 90-95 | + +## Common Errors + +### 5403: "Image transformation failed" +- Verify `width`/`height` ≤ 12000 +- Check `quality` 1-100, `dpr` 1-3 +- Don't combine incompatible options + +### 9413: "Rate limit exceeded" +Implement caching and exponential backoff: +```typescript +for (let i = 0; i < 3; i++) { + try { return await env.IMAGES.input(buffer).transform({...}).output(); } + catch { await new Promise(r => setTimeout(r, 2 ** i * 1000)); } +} +``` + +### 5401: "Image too large" +Pre-process images before upload (max 100MB, 12000×12000px) + +### 5400: "Invalid image format" +Supported: JPEG, PNG, GIF, WebP, AVIF, SVG + +### 401/403: "Unauthorized" +Verify API token has `Cloudflare Images → Edit` permission + +## Limits + +| Resource | Limit | +|----------|-------| +| Max input size | 100MB | +| Max dimensions | 12000×12000px | +| Quality range | 1-100 | +| DPR range | 1-3 | +| API rate limit | ~1200 req/min | + +## AVIF Gotchas + +- **Slower encoding**: First request may have higher latency +- **Browser detection**: +```typescript +const format = /image\/avif/.test(request.headers.get('Accept') || '') ? 'avif' : 'webp'; +``` + +## Anti-Patterns + +```typescript +// ❌ No caching - transforms every request +return env.IMAGES.input(buffer).transform({...}).output().response(); + +// ❌ cover without both dimensions +transform({ width: 800, fit: 'cover' }) + +// ✅ Always set both for cover +transform({ width: 800, height: 600, fit: 'cover' }) + +// ❌ Exposes API token to client +// ✅ Use Direct Creator Upload (patterns.md) +``` + +## Debugging + +```typescript +// Check response headers +console.log('Content-Type:', response.headers.get('Content-Type')); + +// Test with curl +// curl -I "https://imagedelivery.net/{hash}/{id}/width=800,format=avif" + +// Monitor logs +// npx wrangler tail +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/images/patterns.md b/plugins/cloudflare/skills/cloudflare/references/images/patterns.md new file mode 100644 index 0000000..c07bf3c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/images/patterns.md @@ -0,0 +1,115 @@ +# Common Patterns + +## URL Transform Options + +``` +width= height= fit=scale-down|contain|cover|crop|pad +quality=85 format=auto|webp|avif|jpeg|png dpr=2 +gravity=auto|face|left|right|top|bottom sharpen=2 blur=10 +rotate=90|180|270 background=white metadata=none|copyright|keep +``` + +## Responsive Images (srcset) + +```html + +``` + +## Format Negotiation + +```typescript +async fetch(request: Request, env: Env): Promise { + const accept = request.headers.get('Accept') || ''; + const format = /image\/avif/.test(accept) ? 'avif' : /image\/webp/.test(accept) ? 'webp' : 'jpeg'; + return env.IMAGES.input(buffer).transform({ format, quality: 85 }).output().response(); +} +``` + +## Direct Creator Upload + +```typescript +// Backend: Generate upload URL +const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${env.ACCOUNT_ID}/images/v2/direct_upload`, + { method: 'POST', headers: { 'Authorization': `Bearer ${env.API_TOKEN}` }, + body: JSON.stringify({ requireSignedURLs: false, metadata: { userId } }) } +); + +// Frontend: Upload to returned uploadURL +const formData = new FormData(); +formData.append('file', file); +await fetch(result.uploadURL, { method: 'POST', body: formData }); +// Use: https://imagedelivery.net/{hash}/${result.id}/public +``` + +## Transform & Store to R2 + +```typescript +async fetch(request: Request, env: Env): Promise { + const file = (await request.formData()).get('image') as File; + const transformed = await env.IMAGES + .input(await file.arrayBuffer()) + .transform({ width: 800, format: 'avif', quality: 80 }) + .output(); + await env.R2.put(`images/${Date.now()}.avif`, transformed.response().body); + return Response.json({ success: true }); +} +``` + +## Watermarking + +```typescript +const watermark = await env.ASSETS.fetch(new URL('/watermark.png', request.url)); +const result = await env.IMAGES + .input(await image.arrayBuffer()) + .draw(env.IMAGES.input(watermark.body).transform({ width: 100 }), { bottom: 20, right: 20, opacity: 0.7 }) + .transform({ format: 'avif' }) + .output(); +return result.response(); +``` + +## Device-Based Transforms + +```typescript +const ua = request.headers.get('User-Agent') || ''; +const isMobile = /Mobile|Android|iPhone/i.test(ua); +return env.IMAGES.input(buffer) + .transform({ width: isMobile ? 400 : 1200, quality: isMobile ? 75 : 85, format: 'avif' }) + .output().response(); +``` + +## Caching Strategy + +```typescript +async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const cache = caches.default; + let response = await cache.match(request); + if (!response) { + response = await env.IMAGES.input(buffer).transform({ width: 800, format: 'avif' }).output().response(); + response = new Response(response.body, { headers: { ...response.headers, 'Cache-Control': 'public, max-age=86400' } }); + ctx.waitUntil(cache.put(request, response.clone())); + } + return response; +} +``` + +## Batch Processing + +```typescript +const results = await Promise.all(images.map(buffer => + env.IMAGES.input(buffer).transform({ width: 800, fit: 'cover', format: 'avif' }).output() +)); +``` + +## Error Handling + +```typescript +try { + return (await env.IMAGES.input(buffer).transform({ width: 800 }).output()).response(); +} catch (error) { + console.error('Transform failed:', error); + return new Response('Image processing failed', { status: 500 }); +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/kv/README.md b/plugins/cloudflare/skills/cloudflare/references/kv/README.md new file mode 100644 index 0000000..9e43e01 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/kv/README.md @@ -0,0 +1,89 @@ +# Cloudflare Workers KV + +Globally-distributed, eventually-consistent key-value store optimized for high read volume and low latency. + +## Overview + +KV provides: +- Eventual consistency (60s global propagation) +- Read-optimized performance +- 25 MiB value limit per key +- Auto-replication to Cloudflare edge +- Metadata support (1024 bytes) + +**Use cases:** Config storage, user sessions, feature flags, caching, A/B testing + +## When to Use KV + +| Need | Recommendation | +|------|----------------| +| Strong consistency | → [Durable Objects](../durable-objects/) | +| SQL queries | → [D1](../d1/) | +| Object storage (files) | → [R2](../r2/) | +| High read, low write volume | → KV ✅ | +| Sub-10ms global reads | → KV ✅ | + +**Quick comparison:** + +| Feature | KV | D1 | Durable Objects | +|---------|----|----|-----------------| +| Consistency | Eventual | Strong | Strong | +| Read latency | <10ms | ~50ms | <1ms | +| Write limit | 1/s per key | Unlimited | Unlimited | +| Use case | Config, cache | Relational data | Coordination | + +## Quick Start + +```bash +wrangler kv namespace create MY_NAMESPACE +# Add binding to wrangler.jsonc +``` + +```typescript +// Write +await env.MY_KV.put("key", "value", { expirationTtl: 300 }); + +// Read +const value = await env.MY_KV.get("key"); +const json = await env.MY_KV.get("config", "json"); +``` + +## Core Operations + +| Method | Purpose | Returns | +|--------|---------|---------| +| `get(key, type?)` | Single read | `string \| null` | +| `get(keys, type?)` | Bulk read (≤100) | `Map` | +| `put(key, value, options?)` | Write | `Promise` | +| `delete(key)` | Delete | `Promise` | +| `list(options?)` | List keys | `{ keys, list_complete, cursor? }` | +| `getWithMetadata(key)` | Get + metadata | `{ value, metadata }` | + +## Consistency Model + +- **Write visibility:** Immediate in same location, ≤60s globally +- **Read path:** Eventually consistent +- **Write rate:** 1 write/second per key (429 on exceed) + +## Reading Order + +| Task | Files to Read | +|------|---------------| +| Quick start | README → configuration.md | +| Implement feature | README → api.md → patterns.md | +| Debug issues | gotchas.md → api.md | +| Batch operations | api.md (bulk section) → patterns.md | +| Performance tuning | gotchas.md (performance) → patterns.md (caching) | + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc setup, namespace creation, TypeScript types +- [api.md](./api.md) - KV methods, bulk operations, cacheTtl, content types +- [patterns.md](./patterns.md) - Caching, sessions, rate limiting, A/B testing +- [gotchas.md](./gotchas.md) - Eventual consistency, concurrent writes, value limits + +## See Also + +- [workers](../workers/) - Worker runtime for KV access +- [d1](../d1/) - Use D1 for strong consistency needs +- [durable-objects](../durable-objects/) - Strongly consistent alternative diff --git a/plugins/cloudflare/skills/cloudflare/references/kv/api.md b/plugins/cloudflare/skills/cloudflare/references/kv/api.md new file mode 100644 index 0000000..35063f2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/kv/api.md @@ -0,0 +1,160 @@ +# KV API Reference + +## Read Operations + +```typescript +// Single key (string) +const value = await env.MY_KV.get("user:123"); + +// JSON type (auto-parsed) +const config = await env.MY_KV.get("config", "json"); + +// ArrayBuffer for binary +const buffer = await env.MY_KV.get("image", "arrayBuffer"); + +// Stream for large values +const stream = await env.MY_KV.get("large-file", "stream"); + +// With cache TTL (min 60s) +const value = await env.MY_KV.get("key", { type: "text", cacheTtl: 300 }); + +// Bulk get (max 100 keys, counts as 1 operation) +const keys = ["user:1", "user:2", "user:3", "missing:key"]; +const results = await env.MY_KV.get(keys); +// Returns Map + +console.log(results.get("user:1")); // "John" (if exists) +console.log(results.get("missing:key")); // null + +// Process results with null handling +for (const [key, value] of results) { + if (value !== null) { + // Handle found keys + console.log(`${key}: ${value}`); + } +} + +// TypeScript with generics (type-safe JSON parsing) +interface UserProfile { name: string; email: string; } +const profile = await env.USERS.get("user:123", "json"); +// profile is typed as UserProfile | null +if (profile) { + console.log(profile.name); // Type-safe access +} + +// Bulk get with type +const configs = await env.MY_KV.get(["config:app", "config:feature"], "json"); +// Map +``` + +## Write Operations + +```typescript +// Basic put +await env.MY_KV.put("key", "value"); +await env.MY_KV.put("config", JSON.stringify({ theme: "dark" })); + +// With expiration (UNIX timestamp) +await env.MY_KV.put("session", token, { + expiration: Math.floor(Date.now() / 1000) + 3600 +}); + +// With TTL (seconds from now, min 60) +await env.MY_KV.put("cache", data, { expirationTtl: 300 }); + +// With metadata (max 1024 bytes) +await env.MY_KV.put("user:profile", userData, { + metadata: { version: 2, lastUpdated: Date.now() } +}); + +// Combined +await env.MY_KV.put("temp", value, { + expirationTtl: 3600, + metadata: { temporary: true } +}); +``` + +## Get with Metadata + +```typescript +// Single key +const result = await env.MY_KV.getWithMetadata("user:profile"); +// { value: string | null, metadata: any | null } + +if (result.value && result.metadata) { + const { version, lastUpdated } = result.metadata; +} + +// Multiple keys (bulk) +const keys = ["key1", "key2", "key3"]; +const results = await env.MY_KV.getWithMetadata(keys); +// Returns Map + +for (const [key, result] of results) { + if (result.value) { + console.log(`${key}: ${result.value}`); + console.log(`Metadata: ${JSON.stringify(result.metadata)}`); + // cacheStatus field indicates cache hit/miss (when available) + } +} + +// With type +const result = await env.MY_KV.getWithMetadata("user:123", "json"); +// result: { value: UserData | null, metadata: any | null, cacheStatus?: string } +``` + +## Delete Operations + +```typescript +await env.MY_KV.delete("key"); // Always succeeds (even if key missing) +``` + +## List Operations + +```typescript +// List all +const keys = await env.MY_KV.list(); +// { keys: [...], list_complete: boolean, cursor?: string } + +// With prefix +const userKeys = await env.MY_KV.list({ prefix: "user:" }); + +// Pagination +let cursor: string | undefined; +let allKeys = []; +do { + const result = await env.MY_KV.list({ cursor, limit: 1000 }); + allKeys.push(...result.keys); + cursor = result.cursor; +} while (!result.list_complete); +``` + +## Performance Considerations + +### Type Selection + +| Type | Use Case | Performance | +|------|----------|-------------| +| `stream` | Large values (>1MB) | Fastest - no buffering | +| `arrayBuffer` | Binary data | Fast - single allocation | +| `text` | String values | Medium | +| `json` | Objects (parse overhead) | Slowest - parsing cost | + +### Parallel Reads + +```typescript +// Efficient parallel reads with Promise.all() +const [user, settings, cache] = await Promise.all([ + env.USERS.get("user:123", "json"), + env.SETTINGS.get("config:app", "json"), + env.CACHE.get("data:latest") +]); +``` + +## Error Handling + +- **Missing keys:** Return `null` (not an error) +- **Rate limit (429):** Retry with exponential backoff (see gotchas.md) +- **Response too large (413):** Values >25MB fail with 413 error + +See [gotchas.md](./gotchas.md) for detailed error patterns and solutions. diff --git a/plugins/cloudflare/skills/cloudflare/references/kv/configuration.md b/plugins/cloudflare/skills/cloudflare/references/kv/configuration.md new file mode 100644 index 0000000..0aefa5f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/kv/configuration.md @@ -0,0 +1,144 @@ +# KV Configuration + +## Create Namespace + +```bash +wrangler kv namespace create MY_NAMESPACE +# Output: { binding = "MY_NAMESPACE", id = "abc123..." } + +wrangler kv namespace create MY_NAMESPACE --preview # For local dev +``` + +## Workers Binding + +**wrangler.jsonc:** +```jsonc +{ + "kv_namespaces": [ + { + "binding": "MY_KV", + "id": "abc123xyz789" + }, + // Optional: Different namespace for preview/development + { + "binding": "MY_KV", + "preview_id": "preview-abc123" + } + ] +} +``` + +## TypeScript Types + +**env.d.ts:** +```typescript +interface Env { + MY_KV: KVNamespace; + SESSIONS: KVNamespace; + CACHE: KVNamespace; +} +``` + +**worker.ts:** +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + // env.MY_KV is now typed as KVNamespace + const value = await env.MY_KV.get("key"); + return new Response(value || "Not found"); + } +} satisfies ExportedHandler; +``` + +**Type-safe JSON operations:** +```typescript +interface UserProfile { + name: string; + email: string; + role: "admin" | "user"; +} + +const profile = await env.USERS.get("user:123", "json"); +// profile: UserProfile | null (type-safe!) +if (profile) { + console.log(profile.name); // TypeScript knows this is a string +} +``` + +## CLI Operations + +```bash +# Put +wrangler kv key put --binding=MY_KV "key" "value" +wrangler kv key put --binding=MY_KV "key" --path=./file.json --ttl=3600 + +# Get +wrangler kv key get --binding=MY_KV "key" + +# Delete +wrangler kv key delete --binding=MY_KV "key" + +# List +wrangler kv key list --binding=MY_KV --prefix="user:" + +# Bulk operations (max 10,000 keys per file) +wrangler kv bulk put data.json --binding=MY_KV +wrangler kv bulk get keys.json --binding=MY_KV +wrangler kv bulk delete keys.json --binding=MY_KV --force +``` + +## Local Development + +```bash +wrangler dev # Local KV (isolated) +wrangler dev --remote # Remote KV (production) + +# Or in wrangler.jsonc: +# "kv_namespaces": [{ "binding": "MY_KV", "id": "...", "remote": true }] +``` + +## REST API + +### Single Operations + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ + apiEmail: process.env.CLOUDFLARE_EMAIL, + apiKey: process.env.CLOUDFLARE_API_KEY +}); + +// Single key operations +await client.kv.namespaces.values.update(namespaceId, 'key', { + account_id: accountId, + value: 'value', + expiration_ttl: 3600 +}); +``` + +### Bulk Operations + +```typescript +// Bulk update (up to 10,000 keys, max 100MB total) +await client.kv.namespaces.bulkUpdate(namespaceId, { + account_id: accountId, + body: [ + { key: "key1", value: "value1", expiration_ttl: 3600 }, + { key: "key2", value: "value2", metadata: { version: 1 } }, + { key: "key3", value: "value3" } + ] +}); + +// Bulk get (up to 100 keys) +const results = await client.kv.namespaces.bulkGet(namespaceId, { + account_id: accountId, + keys: ["key1", "key2", "key3"] +}); + +// Bulk delete (up to 10,000 keys) +await client.kv.namespaces.bulkDelete(namespaceId, { + account_id: accountId, + keys: ["key1", "key2", "key3"] +}); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/kv/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/kv/gotchas.md new file mode 100644 index 0000000..5ad3213 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/kv/gotchas.md @@ -0,0 +1,131 @@ +# KV Gotchas & Troubleshooting + +## Common Errors + +### "Stale Read After Write" + +**Cause:** Eventual consistency means writes may not be immediately visible in other regions +**Solution:** Don't read immediately after write; return confirmation without reading or use the local value you just wrote. Writes visible immediately in same location, ≤60s globally + +```typescript +// ❌ BAD: Read immediately after write +await env.KV.put("key", "value"); +const value = await env.KV.get("key"); // May be null in other regions! + +// ✅ GOOD: Use the value you just wrote +const newValue = "value"; +await env.KV.put("key", newValue); +return new Response(newValue); // Don't re-read +``` + +### "429 Rate Limit on Concurrent Writes" + +**Cause:** Multiple concurrent writes to same key exceeding 1 write/second limit +**Solution:** Use sequential writes, unique keys for concurrent operations, or implement retry with exponential backoff + +```typescript +async function putWithRetry( + kv: KVNamespace, + key: string, + value: string, + maxAttempts = 5 +): Promise { + let delay = 1000; + for (let i = 0; i < maxAttempts; i++) { + try { + await kv.put(key, value); + return; + } catch (err) { + if (err instanceof Error && err.message.includes("429")) { + if (i === maxAttempts - 1) throw err; + await new Promise(r => setTimeout(r, delay)); + delay *= 2; // Exponential backoff + } else { + throw err; + } + } + } +} +``` + +### "Inefficient Multiple Gets" + +**Cause:** Making multiple individual get() calls instead of bulk operation +**Solution:** Use bulk get with array of keys: `env.USERS.get(["user:1", "user:2", "user:3"])` to reduce to 1 operation + +### "Null Reference Error" + +**Cause:** Attempting to use value without checking for null when key doesn't exist +**Solution:** Always handle null returns - KV returns `null` for missing keys, not undefined + +```typescript +// ❌ BAD: Assumes value exists +const config = await env.KV.get("config", "json"); +return config.theme; // TypeError if null! + +// ✅ GOOD: Null checks +const config = await env.KV.get("config", "json"); +return config?.theme ?? "default"; + +// ✅ GOOD: Early return +const config = await env.KV.get("config", "json"); +if (!config) return new Response("Not found", { status: 404 }); +return new Response(config.theme); +``` + +### "Negative Lookup Caching" + +**Cause:** Keys that don't exist are cached as "not found" for up to 60s +**Solution:** Creating a key after checking won't be visible until cache expires + +```typescript +// Check → create pattern has race condition +const exists = await env.KV.get("key"); // null, cached as "not found" +if (!exists) { + await env.KV.put("key", "value"); + // Next get() may still return null for ~60s due to negative cache +} + +// Alternative: Always assume key may not exist, use defaults +const value = await env.KV.get("key") ?? "default-value"; +``` + +## Performance Tips + +| Scenario | Recommendation | Why | +|----------|----------------|-----| +| Large values (>1MB) | Use `stream` type | Avoids buffering entire value in memory | +| Many small keys | Coalesce into one JSON object | Reduces operations, improves cache hit rate | +| High write volume | Spread across different keys | Avoid 1 write/second per-key limit | +| Cold reads | Increase `cacheTtl` parameter | Reduces latency for frequently-read data | +| Bulk operations | Use array form of get() | Single operation, better performance | + +## Cost Examples + +**Free tier:** +- 100K reads/day = 3M/month ✅ +- 1K writes/day = 30K/month ✅ +- 1GB storage ✅ + +**Example paid workload:** +- 10M reads/month = $5.00 +- 100K writes/month = $0.50 +- 1GB storage = $0.50 +- **Total: ~$6/month** + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Key size | 512 bytes | Maximum key length | +| Value size | 25 MiB | Maximum value; 413 error if exceeded | +| Metadata size | 1024 bytes | Maximum metadata per key | +| cacheTtl minimum | 60s | Minimum cache TTL | +| Write rate per key | 1 write/second | All plans; 429 error if exceeded | +| Propagation time | ≤60s | Global propagation time | +| Bulk get max | 100 keys | Maximum keys per bulk operation | +| Operations per Worker | 1,000 | Per request (bulk counts as 1) | +| Reads pricing | $0.50 per 10M | Per million reads | +| Writes pricing | $5.00 per 1M | Per million writes | +| Deletes pricing | $5.00 per 1M | Per million deletes | +| Storage pricing | $0.50 per GB-month | Per GB per month | diff --git a/plugins/cloudflare/skills/cloudflare/references/kv/patterns.md b/plugins/cloudflare/skills/cloudflare/references/kv/patterns.md new file mode 100644 index 0000000..8386074 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/kv/patterns.md @@ -0,0 +1,196 @@ +# KV Patterns & Best Practices + +## Multi-Tier Caching + +```typescript +// Memory → KV → Origin (3-tier cache) +const memoryCache = new Map(); + +async function getCached(env: Env, key: string): Promise { + const now = Date.now(); + + // L1: Memory cache (fastest) + const cached = memoryCache.get(key); + if (cached && cached.expires > now) { + return cached.data; + } + + // L2: KV cache (fast) + const kvValue = await env.CACHE.get(key, "json"); + if (kvValue) { + memoryCache.set(key, { data: kvValue, expires: now + 60000 }); // 1min in memory + return kvValue; + } + + // L3: Origin (slow) + const origin = await fetch(`https://api.example.com/${key}`).then(r => r.json()); + + // Backfill caches + await env.CACHE.put(key, JSON.stringify(origin), { expirationTtl: 300 }); // 5min in KV + memoryCache.set(key, { data: origin, expires: now + 60000 }); + + return origin; +} +``` + +## API Response Caching + +```typescript +async function getCachedData(env: Env, key: string, fetcher: () => Promise): Promise { + const cached = await env.MY_KV.get(key, "json"); + if (cached) return cached; + + const data = await fetcher(); + await env.MY_KV.put(key, JSON.stringify(data), { expirationTtl: 300 }); + return data; +} + +const apiData = await getCachedData( + env, + "cache:users", + () => fetch("https://api.example.com/users").then(r => r.json()) +); +``` + +## Session Management + +```typescript +interface Session { userId: string; expiresAt: number; } + +async function createSession(env: Env, userId: string): Promise { + const sessionId = crypto.randomUUID(); + const expiresAt = Date.now() + (24 * 60 * 60 * 1000); + + await env.SESSIONS.put( + `session:${sessionId}`, + JSON.stringify({ userId, expiresAt }), + { expirationTtl: 86400, metadata: { createdAt: Date.now() } } + ); + + return sessionId; +} + +async function getSession(env: Env, sessionId: string): Promise { + const data = await env.SESSIONS.get(`session:${sessionId}`, "json"); + if (!data || data.expiresAt < Date.now()) return null; + return data; +} +``` + +## Coalesce Cold Keys + +```typescript +// ❌ BAD: Many individual keys +await env.KV.put("user:123:name", "John"); +await env.KV.put("user:123:email", "john@example.com"); + +// ✅ GOOD: Single coalesced object +await env.USERS.put("user:123:profile", JSON.stringify({ + name: "John", + email: "john@example.com", + role: "admin" +})); + +// Benefits: Hot key cache, single read, reduced operations +// Trade-off: Harder to update individual fields +``` + +## Prefix-Based Namespacing + +```typescript +// Logical partitioning within single namespace +const PREFIXES = { + users: "user:", + sessions: "session:", + cache: "cache:", + features: "feature:" +} as const; + +// Write with prefix +async function setUser(env: Env, id: string, data: any) { + await env.KV.put(`${PREFIXES.users}${id}`, JSON.stringify(data)); +} + +// Read with prefix +async function getUser(env: Env, id: string) { + return await env.KV.get(`${PREFIXES.users}${id}`, "json"); +} + +// List by prefix +async function listUserIds(env: Env): Promise { + const result = await env.KV.list({ prefix: PREFIXES.users }); + return result.keys.map(k => k.name.replace(PREFIXES.users, "")); +} + +// Example hierarchy +"user:123:profile" +"user:123:settings" +"cache:api:users" +"session:abc-def" +"feature:flags:beta" +``` + +## Metadata Versioning + +```typescript +interface VersionedData { + version: number; + data: any; +} + +async function migrateIfNeeded(env: Env, key: string) { + const result = await env.DATA.getWithMetadata(key, "json"); + + if (!result.value) return null; + + const currentVersion = result.metadata?.version || 1; + const targetVersion = 2; + + if (currentVersion < targetVersion) { + // Migrate data format + const migrated = migrate(result.value, currentVersion, targetVersion); + + // Store with new version + await env.DATA.put(key, JSON.stringify(migrated), { + metadata: { version: targetVersion, migratedAt: Date.now() } + }); + + return migrated; + } + + return result.value; +} + +function migrate(data: any, from: number, to: number): any { + if (from === 1 && to === 2) { + // V1 → V2: Rename field + return { ...data, userName: data.name }; + } + return data; +} +``` + +## Error Boundary Pattern + +```typescript +// Resilient get with fallback +async function resilientGet( + env: Env, + key: string, + fallback: T +): Promise { + try { + const value = await env.KV.get(key, "json"); + return value ?? fallback; + } catch (err) { + console.error(`KV error for ${key}:`, err); + return fallback; + } +} + +// Usage +const config = await resilientGet(env, "config:app", { + theme: "light", + maxItems: 10 +}); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/miniflare/README.md b/plugins/cloudflare/skills/cloudflare/references/miniflare/README.md new file mode 100644 index 0000000..82baf7c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/miniflare/README.md @@ -0,0 +1,105 @@ +# Miniflare + +Local simulator for Cloudflare Workers development/testing. Runs Workers in workerd sandbox implementing runtime APIs - no internet required. + +## Features + +- Full-featured: KV, Durable Objects, R2, D1, WebSockets, Queues +- Fully-local: test without internet, instant reload +- TypeScript-native: detailed logging, source maps +- Advanced testing: dispatch events without HTTP, simulate Worker connections + +## When to Use + +**Decision tree for testing Workers:** + +``` +Need to test Workers? +│ +├─ Unit tests for business logic only? +│ └─ getPlatformProxy (Vitest/Jest) → [patterns.md](./patterns.md#getplatformproxy) +│ Fast, no HTTP, direct binding access +│ +├─ Integration tests with full runtime? +│ ├─ Single Worker? +│ │ └─ Miniflare API → [Quick Start](#quick-start) +│ │ Full control, programmatic access +│ │ +│ ├─ Multiple Workers + service bindings? +│ │ └─ Miniflare workers array → [configuration.md](./configuration.md#multiple-workers) +│ │ Shared storage, inter-worker calls +│ │ +│ └─ Vitest test runner integration? +│ └─ vitest-pool-workers → [patterns.md](./patterns.md#vitest-pool-workers) +│ Full Workers env in Vitest +│ +└─ Local dev server? + └─ wrangler dev (not Miniflare) + Hot reload, automatic config +``` + +**Use Miniflare for:** +- Integration tests with full Worker runtime +- Testing bindings/storage locally +- Multiple Workers with service bindings +- Programmatic event dispatch (fetch, queue, scheduled) + +**Use getPlatformProxy for:** +- Fast unit tests of business logic +- Testing without HTTP overhead +- Vitest/Jest environments + +**Use Wrangler for:** +- Local development workflow +- Production deployments + +## Setup + +```bash +npm i -D miniflare +``` + +Requires ES modules in `package.json`: +```json +{"type": "module"} +``` + +## Quick Start + +```js +import { Miniflare } from "miniflare"; + +const mf = new Miniflare({ + modules: true, + script: ` + export default { + async fetch(request, env, ctx) { + return new Response("Hello Miniflare!"); + } + } + `, +}); + +const res = await mf.dispatchFetch("http://localhost:8787/"); +console.log(await res.text()); // Hello Miniflare! +await mf.dispose(); +``` + +## Reading Order + +**New to Miniflare?** Start here: +1. [Quick Start](#quick-start) - Running in 2 minutes +2. [When to Use](#when-to-use) - Choose your testing approach +3. [patterns.md](./patterns.md) - Testing patterns (getPlatformProxy, Vitest, node:test) +4. [configuration.md](./configuration.md) - Configure bindings, storage, multiple workers + +**Troubleshooting:** +- [gotchas.md](./gotchas.md) - Common errors and debugging + +**API reference:** +- [api.md](./api.md) - Complete method reference + +## See Also +- [wrangler](../wrangler/) - CLI tool that embeds Miniflare for `wrangler dev` +- [workerd](../workerd/) - Runtime that powers Miniflare +- [workers](../workers/) - Workers runtime API documentation diff --git a/plugins/cloudflare/skills/cloudflare/references/miniflare/api.md b/plugins/cloudflare/skills/cloudflare/references/miniflare/api.md new file mode 100644 index 0000000..e4df4d7 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/miniflare/api.md @@ -0,0 +1,187 @@ +# Programmatic API + +## Miniflare Class + +```typescript +class Miniflare { + constructor(options: MiniflareOptions); + + // Lifecycle + ready: Promise; // Resolves when server ready, returns URL + dispose(): Promise; // Cleanup resources + setOptions(options: MiniflareOptions): Promise; // Reload config + + // Event dispatching + dispatchFetch(url: string | URL | Request, init?: RequestInit): Promise; + getWorker(name?: string): Promise; + + // Bindings access + getBindings>(name?: string): Promise; + getCf(name?: string): Promise; + getKVNamespace(name: string): Promise; + getR2Bucket(name: string): Promise; + getDurableObjectNamespace(name: string): Promise; + getDurableObjectStorage(id: DurableObjectId): Promise; + getD1Database(name: string): Promise; + getCaches(): Promise; + getQueueProducer(name: string): Promise; + + // Debugging + getInspectorURL(): Promise; // Chrome DevTools inspector URL +} +``` + +## Event Dispatching + +**Fetch (no HTTP server):** +```js +const res = await mf.dispatchFetch("http://localhost:8787/path", { + method: "POST", + headers: { "Authorization": "Bearer token" }, + body: JSON.stringify({ data: "value" }), +}); +``` + +**Custom Host routing:** +```js +const res = await mf.dispatchFetch("http://localhost:8787/", { + headers: { "Host": "api.example.com" }, +}); +``` + +**Scheduled:** +```js +const worker = await mf.getWorker(); +const result = await worker.scheduled({ cron: "30 * * * *" }); +// result: { outcome: "ok", noRetry: false } +``` + +**Queue:** +```js +const worker = await mf.getWorker(); +const result = await worker.queue("queue-name", [ + { id: "msg1", timestamp: new Date(), body: "data", attempts: 1 }, +]); +// result: { outcome: "ok", retryAll: false, ackAll: false, ... } +``` + +## Bindings Access + +**Environment variables:** +```js +// Basic usage +const bindings = await mf.getBindings(); +console.log(bindings.SECRET_KEY); + +// With type safety (recommended): +interface Env { + SECRET_KEY: string; + API_URL: string; + KV: KVNamespace; +} +const env = await mf.getBindings(); +env.SECRET_KEY; // string (typed!) +env.KV.get("key"); // KVNamespace methods available +``` + +**Request.cf object:** +```js +const cf = await mf.getCf(); +console.log(cf?.colo); // "DFW" +console.log(cf?.country); // "US" +``` + +**KV:** +```js +const ns = await mf.getKVNamespace("TEST_NAMESPACE"); +await ns.put("key", "value"); +const value = await ns.get("key"); +``` + +**R2:** +```js +const bucket = await mf.getR2Bucket("BUCKET"); +await bucket.put("file.txt", "content"); +const object = await bucket.get("file.txt"); +``` + +**Durable Objects:** +```js +const ns = await mf.getDurableObjectNamespace("COUNTER"); +const id = ns.idFromName("test"); +const stub = ns.get(id); +const res = await stub.fetch("http://localhost/"); + +// Access storage directly: +const storage = await mf.getDurableObjectStorage(id); +await storage.put("key", "value"); +``` + +**D1:** +```js +const db = await mf.getD1Database("DB"); +await db.exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)`); +await db.prepare("INSERT INTO users (name) VALUES (?)").bind("Alice").run(); +``` + +**Cache:** +```js +const caches = await mf.getCaches(); +const defaultCache = caches.default; +await defaultCache.put("http://example.com", new Response("cached")); +``` + +**Queue producer:** +```js +const producer = await mf.getQueueProducer("QUEUE"); +await producer.send({ body: "message data" }); +``` + +## Lifecycle + +**Reload:** +```js +await mf.setOptions({ + scriptPath: "worker.js", + bindings: { VERSION: "2.0" }, +}); +``` + +**Watch (manual):** +```js +import { watch } from "fs"; + +const config = { scriptPath: "worker.js" }; +const mf = new Miniflare(config); + +watch("worker.js", async () => { + console.log("Reloading..."); + await mf.setOptions(config); +}); +``` + +**Cleanup:** +```js +await mf.dispose(); +``` + +## Debugging + +**Inspector URL for DevTools:** +```js +const url = await mf.getInspectorURL(); +console.log(`DevTools: ${url}`); +// Open in Chrome DevTools for breakpoints, profiling +``` + +**Wait for server ready:** +```js +const mf = new Miniflare({ scriptPath: "worker.js" }); +const url = await mf.ready; // Promise +console.log(`Server running at ${url}`); // http://127.0.0.1:8787 + +// Note: dispatchFetch() waits automatically, no need to await ready +const res = await mf.dispatchFetch("http://localhost/"); // Works immediately +``` + +See [configuration.md](./configuration.md) for all constructor options. diff --git a/plugins/cloudflare/skills/cloudflare/references/miniflare/configuration.md b/plugins/cloudflare/skills/cloudflare/references/miniflare/configuration.md new file mode 100644 index 0000000..b269b24 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/miniflare/configuration.md @@ -0,0 +1,173 @@ +# Configuration + +## Script Loading + +```js +// Inline +new Miniflare({ modules: true, script: `export default { ... }` }); + +// File-based +new Miniflare({ scriptPath: "worker.js" }); + +// Multi-module +new Miniflare({ + scriptPath: "src/index.js", + modules: true, + modulesRules: [ + { type: "ESModule", include: ["**/*.js"] }, + { type: "Text", include: ["**/*.txt"] }, + ], +}); +``` + +## Compatibility + +```js +new Miniflare({ + compatibilityDate: "2026-01-01", // Use recent date for latest features + compatibilityFlags: [ + "nodejs_compat", // Node.js APIs (process, Buffer, etc) + "streams_enable_constructors", // Stream constructors + ], + upstream: "https://example.com", // Fallback for unhandled requests +}); +``` + +**Critical:** Use `compatibilityDate: "2026-01-01"` or latest to match production runtime. Old dates limit available APIs. + +## HTTP Server & Request.cf + +```js +new Miniflare({ + port: 8787, // Default: 8787 + host: "127.0.0.1", + https: true, // Self-signed cert + liveReload: true, // Auto-reload HTML + + cf: true, // Fetch live Request.cf data (cached) + // cf: "./cf.json", // Or load from file + // cf: { colo: "DFW" }, // Or inline mock +}); +``` + +**Note:** For tests, use `dispatchFetch()` (no port conflicts). + +## Storage Bindings + +```js +new Miniflare({ + // KV + kvNamespaces: ["TEST_NAMESPACE", "CACHE"], + kvPersist: "./kv-data", // Optional: persist to disk + + // R2 + r2Buckets: ["BUCKET", "IMAGES"], + r2Persist: "./r2-data", + + // Durable Objects + modules: true, + durableObjects: { + COUNTER: "Counter", // className + API_OBJECT: { className: "ApiObject", scriptName: "api-worker" }, + }, + durableObjectsPersist: "./do-data", + + // D1 + d1Databases: ["DB"], + d1Persist: "./d1-data", + + // Cache + cache: true, // Default + cachePersist: "./cache-data", +}); +``` + +## Bindings + +```js +new Miniflare({ + // Environment variables + bindings: { + SECRET_KEY: "my-secret-value", + API_URL: "https://api.example.com", + DEBUG: true, + }, + + // Other bindings + wasmBindings: { ADD_MODULE: "./add.wasm" }, + textBlobBindings: { TEXT: "./data.txt" }, + queueProducers: ["QUEUE"], +}); +``` + +## Multiple Workers + +```js +new Miniflare({ + workers: [ + { + name: "main", + kvNamespaces: { DATA: "shared" }, + serviceBindings: { API: "api-worker" }, + script: `export default { ... }`, + }, + { + name: "api-worker", + kvNamespaces: { DATA: "shared" }, // Shared storage + script: `export default { ... }`, + }, + ], +}); +``` + +**With routing:** +```js +workers: [ + { name: "api", scriptPath: "./api.js", routes: ["api.example.com/*"] }, + { name: "web", scriptPath: "./web.js", routes: ["example.com/*"] }, +], +``` + +## Logging & Performance + +```js +import { Log, LogLevel } from "miniflare"; + +new Miniflare({ + log: new Log(LogLevel.DEBUG), // DEBUG | INFO | WARN | ERROR | NONE + scriptTimeout: 30000, // CPU limit (ms) + workersConcurrencyLimit: 10, // Max concurrent workers +}); +``` + +## Workers Sites + +```js +new Miniflare({ + sitePath: "./public", + siteInclude: ["**/*.html", "**/*.css"], + siteExclude: ["**/*.map"], +}); +``` + +## From wrangler.toml + +Miniflare doesn't auto-read `wrangler.toml`: + +```toml +# wrangler.toml +name = "my-worker" +main = "src/index.ts" +compatibility_date = "2026-01-01" +[[kv_namespaces]] +binding = "KV" +``` + +```js +// Miniflare equivalent +new Miniflare({ + scriptPath: "src/index.ts", + compatibilityDate: "2026-01-01", + kvNamespaces: ["KV"], +}); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/miniflare/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/miniflare/gotchas.md new file mode 100644 index 0000000..dfcd157 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/miniflare/gotchas.md @@ -0,0 +1,160 @@ +# Gotchas & Troubleshooting + +## Miniflare Limitations + +**Not supported:** +- Analytics Engine (use mocks) +- Cloudflare Images/Stream +- Browser Rendering API +- Tail Workers +- Workers for Platforms (partial support) + +**Behavior differences from production:** +- Runs workerd locally, not Cloudflare edge +- Storage is local (filesystem/memory), not distributed +- `Request.cf` is cached/mocked, not real edge data +- Performance differs from edge +- Caching implementation may vary slightly + +## Common Errors + +### "Cannot find module" +**Cause:** Module path wrong or `modulesRules` not configured +**Solution:** +```js +new Miniflare({ + modules: true, + modulesRules: [{ type: "ESModule", include: ["**/*.js"] }], +}); +``` + +### "Data not persisting" +**Cause:** Persist paths are files, not directories +**Solution:** +```js +kvPersist: "./data/kv", // Directory, not file +``` + +### "Cannot run TypeScript" +**Cause:** Miniflare doesn't transpile TypeScript +**Solution:** Build first with esbuild/tsc, then run compiled JS + +### "`request.cf` is undefined" +**Cause:** CF data not configured +**Solution:** +```js +new Miniflare({ cf: true }); // Or cf: "./cf.json" +``` + +### "EADDRINUSE" port conflict +**Cause:** Multiple instances using same port +**Solution:** Use `dispatchFetch()` (no HTTP server) or `port: 0` for auto-assign + +### "Durable Object not found" +**Cause:** Class export doesn't match config name +**Solution:** +```js +export class Counter {} // Must match +new Miniflare({ durableObjects: { COUNTER: "Counter" } }); +``` + +## Debugging + +**Enable verbose logging:** +```js +import { Log, LogLevel } from "miniflare"; +new Miniflare({ log: new Log(LogLevel.DEBUG) }); +``` + +**Chrome DevTools:** +```js +const url = await mf.getInspectorURL(); +console.log(`DevTools: ${url}`); // Open in Chrome +``` + +**Inspect bindings:** +```js +const env = await mf.getBindings(); +console.log(Object.keys(env)); +``` + +**Verify storage:** +```js +const ns = await mf.getKVNamespace("TEST"); +const { keys } = await ns.list(); +``` + +## Best Practices + +**✓ Do:** +- Use `dispatchFetch()` for tests (no HTTP server) +- In-memory storage for CI (omit persist options) +- New instances per test for isolation +- Type-safe bindings with interfaces +- `await mf.dispose()` in cleanup + +**✗ Avoid:** +- HTTP server in tests +- Shared instances without cleanup +- Old compatibility dates (use 2026+) + +## Migration Guides + +### From Miniflare 2.x to 3+ + +Breaking changes in v3+: + +| v2 | v3+ | +|----|-----| +| `getBindings()` sync | `getBindings()` returns Promise | +| `ready` is void | `ready` returns `Promise` | +| service-worker-mock | Built on workerd | +| Different options | Restructured constructor | + +**Example migration:** +```js +// v2 +const bindings = mf.getBindings(); +mf.ready; // void + +// v3+ +const bindings = await mf.getBindings(); +const url = await mf.ready; // Promise +``` + +### From unstable_dev to Miniflare + +```js +// Old (deprecated) +import { unstable_dev } from "wrangler"; +const worker = await unstable_dev("src/index.ts"); + +// New +import { Miniflare } from "miniflare"; +const mf = new Miniflare({ scriptPath: "src/index.ts" }); +``` + +### From Wrangler Dev + +Miniflare doesn't auto-read `wrangler.toml`: + +```js +// Translate manually: +new Miniflare({ + scriptPath: "dist/worker.js", + compatibilityDate: "2026-01-01", + kvNamespaces: ["KV"], + bindings: { API_KEY: process.env.API_KEY }, +}); +``` + +## Resource Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| CPU time | 30s default | Configurable via `scriptTimeout` | +| Storage | Filesystem | Performance varies by disk | +| Memory | System dependent | No artificial limits | +| Request.cf | Cached/mocked | Not live edge data | + +See [patterns.md](./patterns.md) for testing examples. diff --git a/plugins/cloudflare/skills/cloudflare/references/miniflare/patterns.md b/plugins/cloudflare/skills/cloudflare/references/miniflare/patterns.md new file mode 100644 index 0000000..c89c3a5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/miniflare/patterns.md @@ -0,0 +1,181 @@ +# Testing Patterns + +## Choosing a Testing Approach + +| Approach | Use Case | Speed | Setup | Runtime | +|----------|----------|-------|-------|---------| +| **getPlatformProxy** | Unit tests, logic testing | Fast | Low | Miniflare | +| **Miniflare API** | Integration tests, full control | Medium | Medium | Miniflare | +| **vitest-pool-workers** | Vitest runner integration | Medium | Medium | workerd | + +**Quick guide:** +- Unit tests → getPlatformProxy +- Integration tests → Miniflare API +- Vitest workflows → vitest-pool-workers + +## getPlatformProxy + +Lightweight unit testing - provides bindings without full Worker runtime. + +```js +// vitest.config.js +export default { test: { environment: "node" } }; +``` + +```js +import { env } from "cloudflare:test"; +import { describe, it, expect } from "vitest"; + +describe("Business logic", () => { + it("processes data with KV", async () => { + await env.KV.put("test", "value"); + expect(await env.KV.get("test")).toBe("value"); + }); +}); +``` + +**Pros:** Fast, simple +**Cons:** No full runtime, can't test fetch handler + +## vitest-pool-workers + +Full Workers runtime in Vitest. Reads `wrangler.toml`. + +```bash +npm i -D @cloudflare/vitest-pool-workers +``` + +```js +// vitest.config.js +import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config"; + +export default defineWorkersConfig({ + test: { + poolOptions: { workers: { wrangler: { configPath: "./wrangler.toml" } } }, + }, +}); +``` + +```js +import { env, SELF } from "cloudflare:test"; +import { it, expect } from "vitest"; + +it("handles fetch", async () => { + const res = await SELF.fetch("http://example.com/"); + expect(res.status).toBe(200); +}); +``` + +**Pros:** Full runtime, uses wrangler.toml +**Cons:** Requires Wrangler config + +## Miniflare API (node:test) + +```js +import assert from "node:assert"; +import test, { after, before } from "node:test"; +import { Miniflare } from "miniflare"; + +let mf; +before(() => { + mf = new Miniflare({ scriptPath: "src/index.js", kvNamespaces: ["TEST_KV"] }); +}); + +test("fetch", async () => { + const res = await mf.dispatchFetch("http://localhost/"); + assert.strictEqual(await res.text(), "Hello"); +}); + +after(() => mf.dispose()); +``` + +## Testing Durable Objects & Events + +```js +// Durable Objects +const ns = await mf.getDurableObjectNamespace("COUNTER"); +const stub = ns.get(ns.idFromName("test-counter")); +await stub.fetch("http://localhost/increment"); + +// Direct storage +const storage = await mf.getDurableObjectStorage(ns.idFromName("test-counter")); +const count = await storage.get("count"); + +// Queue +const worker = await mf.getWorker(); +await worker.queue("my-queue", [ + { id: "msg1", timestamp: new Date(), body: { userId: 123 }, attempts: 1 }, +]); + +// Scheduled +await worker.scheduled({ cron: "0 0 * * *" }); +``` + +## Test Isolation & Mocking + +```js +// Per-test isolation +beforeEach(() => { mf = new Miniflare({ kvNamespaces: ["TEST"] }); }); +afterEach(() => mf.dispose()); + +// Mock external APIs +new Miniflare({ + workers: [ + { name: "main", serviceBindings: { API: "mock-api" }, script: `...` }, + { name: "mock-api", script: `export default { async fetch() { return Response.json({mock: true}); } }` }, + ], +}); +``` + +## Type Safety + +```ts +import type { KVNamespace } from "@cloudflare/workers-types"; + +interface Env { + KV: KVNamespace; + API_KEY: string; +} + +const env = await mf.getBindings(); +await env.KV.put("key", "value"); // Typed! + +export default { + async fetch(req: Request, env: Env) { + return new Response(await env.KV.get("key")); + } +} satisfies ExportedHandler; +``` + +## WebSocket Testing + +```js +const res = await mf.dispatchFetch("http://localhost/ws", { + headers: { Upgrade: "websocket" }, +}); +assert.strictEqual(res.status, 101); +``` + +## Migration from unstable_dev + +```js +// Old (deprecated) +import { unstable_dev } from "wrangler"; +const worker = await unstable_dev("src/index.ts"); + +// New +import { Miniflare } from "miniflare"; +const mf = new Miniflare({ scriptPath: "src/index.ts" }); +``` + +## CI/CD Tips + +```js +// In-memory storage (faster) +new Miniflare({ kvNamespaces: ["TEST"] }); // No persist = in-memory + +// Use dispatchFetch (no port conflicts) +await mf.dispatchFetch("http://localhost/"); +``` + +See [gotchas.md](./gotchas.md) for troubleshooting. diff --git a/plugins/cloudflare/skills/cloudflare/references/network-interconnect/README.md b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/README.md new file mode 100644 index 0000000..e337f1b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/README.md @@ -0,0 +1,99 @@ +# Cloudflare Network Interconnect (CNI) + +Private, high-performance connectivity to Cloudflare's network. **Enterprise-only**. + +## Connection Types + +**Direct**: Physical fiber in shared datacenter. 10/100 Gbps. You order cross-connect. + +**Partner**: Virtual via Console Connect, Equinix, Megaport, etc. Managed via partner SDN. + +**Cloud**: AWS Direct Connect or GCP Cloud Interconnect. Magic WAN only. + +## Dataplane Versions + +**v1 (Classic)**: GRE tunnel support, VLAN/BFD/LACP, asymmetric MTU (1500↓/1476↑), peering support. + +**v2 (Beta)**: No GRE, 1500 MTU both ways, no VLAN/BFD/LACP yet, ECMP instead. + +## Use Cases + +- **Magic Transit DSR**: DDoS protection, egress via ISP (v1/v2) +- **Magic Transit + Egress**: DDoS + egress via CF (v1/v2) +- **Magic WAN + Zero Trust**: Private backbone (v1 needs GRE, v2 native) +- **Peering**: Public routes at PoP (v1 only) +- **App Security**: WAF/Cache/LB (v1/v2 over Magic Transit) + +## Prerequisites + +- Enterprise plan +- IPv4 /24+ or IPv6 /48+ prefixes +- BGP ASN for v1 +- See [locations PDF](https://developers.cloudflare.com/network-interconnect/static/cni-locations-2026-01.pdf) + +## Specs + +- /31 point-to-point subnets +- 10km max optical distance +- 10G: 10GBASE-LR single-mode +- 100G: 100GBASE-LR4 single-mode +- **No SLA** (free service) +- Backup Internet required + +## Throughput + +| Direction | 10G | 100G | +|-----------|-----|------| +| CF → Customer | 10 Gbps | 100 Gbps | +| Customer → CF (peering) | 10 Gbps | 100 Gbps | +| Customer → CF (Magic) | 1 Gbps/tunnel or CNI | 1 Gbps/tunnel or CNI | + +## Timeline + +2-4 weeks typical. Steps: request → config review → order connection → configure → test → enable health checks → activate → monitor. + +## In This Reference +- [configuration.md](./configuration.md) - BGP, routing, setup +- [api.md](./api.md) - API endpoints, SDKs +- [patterns.md](./patterns.md) - HA, hybrid cloud, failover +- [gotchas.md](./gotchas.md) - Troubleshooting, limits + +## Reading Order by Task + +| Task | Files to Load | +|------|---------------| +| Initial setup | README → configuration.md → api.md | +| Create interconnect via API | api.md → gotchas.md | +| Design HA architecture | patterns.md → README | +| Troubleshoot connection | gotchas.md → configuration.md | +| Cloud integration (AWS/GCP) | configuration.md → patterns.md | +| Monitor + alerts | configuration.md | + +## Automation Boundary + +**API-Automatable:** +- List/create/delete interconnects (Direct, Partner) +- List available slots +- Get interconnect status +- Download LOA PDF +- Create/update CNI objects (BGP config) +- Query settings + +**Requires Account Team:** +- Initial request approval +- AWS Direct Connect setup (send LOA+VLAN to CF) +- GCP Cloud Interconnect final activation +- Partner interconnect acceptance (Equinix, Megaport) +- VLAN assignment (v1) +- Configuration document generation (v1) +- Escalations + troubleshooting support + +**Cannot Be Automated:** +- Physical cross-connect installation (Direct) +- Partner portal operations (virtual circuit ordering) +- AWS/GCP portal operations +- Maintenance window coordination + +## See Also +- [tunnel](../tunnel/) - Alternative for private network connectivity +- [spectrum](../spectrum/) - Layer 4 proxy for TCP/UDP traffic diff --git a/plugins/cloudflare/skills/cloudflare/references/network-interconnect/api.md b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/api.md new file mode 100644 index 0000000..85e5e12 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/api.md @@ -0,0 +1,199 @@ +# CNI API Reference + +See [README.md](README.md) for overview. + +## Base + +``` +https://api.cloudflare.com/client/v4 +Auth: Authorization: Bearer +``` + +## SDK Namespaces + +**Primary (recommended):** +```typescript +client.networkInterconnects.interconnects.* +client.networkInterconnects.cnis.* +client.networkInterconnects.slots.* +``` + +**Alternate (deprecated):** +```typescript +client.magicTransit.cfInterconnects.* +``` + +Use `networkInterconnects` namespace for all new code. + +## Interconnects + +```http +GET /accounts/{account_id}/cni/interconnects # Query: page, per_page +POST /accounts/{account_id}/cni/interconnects # Query: validate_only=true (optional) +GET /accounts/{account_id}/cni/interconnects/{icon} +GET /accounts/{account_id}/cni/interconnects/{icon}/status +GET /accounts/{account_id}/cni/interconnects/{icon}/loa # Returns PDF +DELETE /accounts/{account_id}/cni/interconnects/{icon} +``` + +**Create Body:** `account`, `slot_id`, `type`, `facility`, `speed`, `name`, `description` +**Status Values:** `active` | `healthy` | `unhealthy` | `pending` | `down` + +**Response Example:** +```json +{"result": [{"id": "icon_abc", "name": "prod", "type": "direct", "facility": "EWR1", "speed": "10G", "status": "active"}]} +``` + +## CNI Objects (BGP config) + +```http +GET /accounts/{account_id}/cni/cnis +POST /accounts/{account_id}/cni/cnis +GET /accounts/{account_id}/cni/cnis/{cni} +PUT /accounts/{account_id}/cni/cnis/{cni} +DELETE /accounts/{account_id}/cni/cnis/{cni} +``` + +Body: `account`, `cust_ip`, `cf_ip`, `bgp_asn`, `bgp_password`, `vlan` + +## Slots + +```http +GET /accounts/{account_id}/cni/slots +GET /accounts/{account_id}/cni/slots/{slot} +``` + +Query: `facility`, `occupied`, `speed` + +## Health Checks + +Configure via Magic Transit/WAN tunnel endpoints (CNI v2). + +```typescript +await client.magicTransit.tunnels.update(accountId, tunnelId, { + health_check: { enabled: true, target: '192.0.2.1', rate: 'high', type: 'request' }, +}); +``` + +Rates: `high` | `medium` | `low`. Types: `request` | `reply`. See [Magic Transit docs](https://developers.cloudflare.com/magic-transit/how-to/configure-tunnel-endpoints/#add-tunnels). + +## Settings + +```http +GET /accounts/{account_id}/cni/settings +PUT /accounts/{account_id}/cni/settings +``` + +Body: `default_asn` + +## TypeScript SDK + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: process.env.CF_TOKEN }); + +// List +await client.networkInterconnects.interconnects.list({ account_id: id }); + +// Create with validation +await client.networkInterconnects.interconnects.create({ + account_id: id, + account: id, + slot_id: 'slot_abc', + type: 'direct', + facility: 'EWR1', + speed: '10G', + name: 'prod-interconnect', +}, { + query: { validate_only: true }, // Dry-run validation +}); + +// Create without validation +await client.networkInterconnects.interconnects.create({ + account_id: id, + account: id, + slot_id: 'slot_abc', + type: 'direct', + facility: 'EWR1', + speed: '10G', + name: 'prod-interconnect', +}); + +// Status +await client.networkInterconnects.interconnects.get(accountId, iconId); + +// LOA (use fetch) +const res = await fetch(`https://api.cloudflare.com/client/v4/accounts/${id}/cni/interconnects/${iconId}/loa`, { + headers: { Authorization: `Bearer ${token}` }, +}); +await fs.writeFile('loa.pdf', Buffer.from(await res.arrayBuffer())); + +// CNI object +await client.networkInterconnects.cnis.create({ + account_id: id, + account: id, + cust_ip: '192.0.2.1/31', + cf_ip: '192.0.2.0/31', + bgp_asn: 65000, + vlan: 100, +}); + +// Slots (filter by facility and speed) +await client.networkInterconnects.slots.list({ + account_id: id, + occupied: false, + facility: 'EWR1', + speed: '10G', +}); +``` + +## Python SDK + +```python +from cloudflare import Cloudflare + +client = Cloudflare(api_token=os.environ["CF_TOKEN"]) + +# List, create, status (same pattern as TypeScript) +client.network_interconnects.interconnects.list(account_id=id) +client.network_interconnects.interconnects.create(account_id=id, account=id, slot_id="slot_abc", type="direct", facility="EWR1", speed="10G") +client.network_interconnects.interconnects.get(account_id=id, icon=icon_id) + +# CNI objects and slots +client.network_interconnects.cnis.create(account_id=id, cust_ip="192.0.2.1/31", cf_ip="192.0.2.0/31", bgp_asn=65000) +client.network_interconnects.slots.list(account_id=id, occupied=False) +``` + +## cURL + +```bash +# List interconnects +curl "https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/cni/interconnects" \ + -H "Authorization: Bearer ${CF_TOKEN}" + +# Create interconnect +curl -X POST "https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/cni/interconnects?validate_only=true" \ + -H "Authorization: Bearer ${CF_TOKEN}" -H "Content-Type: application/json" \ + -d '{"account": "id", "slot_id": "slot_abc", "type": "direct", "facility": "EWR1", "speed": "10G"}' + +# LOA PDF +curl "https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/cni/interconnects/${ICON_ID}/loa" \ + -H "Authorization: Bearer ${CF_TOKEN}" --output loa.pdf +``` + +## Not Available via API + +**Missing Capabilities:** +- BGP session state query (use Dashboard or BGP logs) +- Bandwidth utilization metrics (use external monitoring) +- Traffic statistics per interconnect +- Historical uptime/downtime data +- Light level readings (contact account team) +- Maintenance window scheduling (notifications only) + +## Resources + +- [API Docs](https://developers.cloudflare.com/api/resources/network_interconnects/) +- [TypeScript SDK](https://github.com/cloudflare/cloudflare-typescript) +- [Python SDK](https://github.com/cloudflare/cloudflare-python) diff --git a/plugins/cloudflare/skills/cloudflare/references/network-interconnect/configuration.md b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/configuration.md new file mode 100644 index 0000000..0f1005c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/configuration.md @@ -0,0 +1,114 @@ +# CNI Configuration + +See [README.md](README.md) for overview. + +## Workflow (2-4 weeks) + +1. **Submit request** (Week 1): Contact account team, provide type/location/use case +2. **Review config** (Week 1-2, v1 only): Approve IP/VLAN/spec doc +3. **Order connection** (Week 2-3): + - **Direct**: Get LOA, order cross-connect from facility + - **Partner**: Order virtual circuit in partner portal + - **Cloud**: Order Direct Connect/Cloud Interconnect, send LOA+VLAN to CF +4. **Configure** (Week 3): Both sides configure per doc +5. **Test** (Week 3-4): Ping, verify BGP, check routes +6. **Health checks** (Week 4): Configure [Magic Transit](https://developers.cloudflare.com/magic-transit/how-to/configure-tunnel-endpoints/#add-tunnels) or [Magic WAN](https://developers.cloudflare.com/magic-wan/configuration/manually/how-to/configure-tunnel-endpoints/#add-tunnels) health checks +7. **Activate** (Week 4): Route traffic, verify flow +8. **Monitor**: Enable [maintenance notifications](https://developers.cloudflare.com/network-interconnect/monitoring-and-alerts/#enable-cloudflare-status-maintenance-notification) + +## BGP Configuration + +**v1 Requirements:** +- BGP ASN (provide during setup) +- /31 subnet for peering +- Optional: BGP password + +**v2:** Simplified, less BGP config needed. + +**BGP over CNI (Dec 2024):** Magic WAN/Transit can now peer BGP directly over CNI v2 (no GRE tunnel required). + +**Example v1 BGP:** +``` +Router ID: 192.0.2.1 +Peer IP: 192.0.2.0 +Remote ASN: 13335 +Local ASN: 65000 +Password: [optional] +VLAN: 100 +``` + +## Cloud Interconnect Setup + +### AWS Direct Connect (Beta) + +**Requirements:** Magic WAN, AWS Dedicated Direct Connect 1/10 Gbps. + +**Process:** +1. Contact CF account team +2. Choose location +3. Order in AWS portal +4. AWS provides LOA + VLAN ID +5. Send to CF account team +6. Wait ~4 weeks + +**Post-setup:** Add [static routes](https://developers.cloudflare.com/magic-wan/configuration/manually/how-to/configure-routes/#configure-static-routes) to Magic WAN. Enable [bidirectional health checks](https://developers.cloudflare.com/magic-wan/configuration/manually/how-to/configure-tunnel-endpoints/#legacy-bidirectional-health-checks). + +### GCP Cloud Interconnect (Beta) + +**Setup via Dashboard:** +1. Interconnects → Create → Cloud Interconnect → Google +2. Provide name, MTU (match GCP VLAN attachment), speed (50M-50G granular options available for partner interconnects) +3. Enter VLAN attachment pairing key +4. Confirm order + +**Routing to GCP:** Add [static routes](https://developers.cloudflare.com/magic-wan/configuration/manually/how-to/configure-routes/#configure-static-routes). BGP routes from GCP Cloud Router **ignored**. + +**Routing to CF:** Configure [custom learned routes](https://cloud.google.com/network-connectivity/docs/router/how-to/configure-custom-learned-routes) in Cloud Router. Request prefixes from CF account team. + +## Monitoring + +**Dashboard Status:** + +| Status | Meaning | +|--------|---------| +| **Healthy** | Link operational, traffic flowing, health checks passing | +| **Active** | Link up, sufficient light, Ethernet negotiated | +| **Unhealthy** | Link down, no/low light (<-20 dBm), can't negotiate | +| **Pending** | Cross-connect incomplete, device unresponsive, RX/TX swapped | +| **Down** | Physical link down, no connectivity | + +**Alerts:** + +**CNI Connection Maintenance** (Magic Networking only): +``` +Dashboard → Notifications → Add +Product: Cloudflare Network Interconnect +Type: Connection Maintenance Alert +``` +Warnings up to 2 weeks advance. 6hr delay for new additions. + +**Cloudflare Status Maintenance** (entire PoP): +``` +Dashboard → Notifications → Add +Product: Cloudflare Status +Filter PoPs: gru,fra,lhr +``` + +**Find PoP code:** +``` +Dashboard → Magic Transit/WAN → Configuration → Interconnects +Select CNI → Note Data Center (e.g., "gru-b") +Use first 3 letters: "gru" +``` + +## Best Practices + +**Critical config-specific practices:** +- /31 subnets required for BGP +- BGP passwords recommended +- BFD for fast failover (v1 only) +- Test ping connectivity before BGP +- Enable maintenance notifications immediately after activation +- Monitor status programmatically via API + +For design patterns, HA architecture, and security best practices, see [patterns.md](./patterns.md). diff --git a/plugins/cloudflare/skills/cloudflare/references/network-interconnect/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/gotchas.md new file mode 100644 index 0000000..9880807 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/gotchas.md @@ -0,0 +1,165 @@ +# CNI Gotchas & Troubleshooting + +## Common Errors + +### "Status: Pending" + +**Cause:** Cross-connect not installed, RX/TX fibers reversed, wrong fiber type, or low light levels +**Solution:** +1. Verify cross-connect installed +2. Check fiber at patch panel +3. Swap RX/TX fibers +4. Check light with optical power meter (target > -20 dBm) +5. Contact account team + +### "Status: Unhealthy" + +**Cause:** Physical issue, low light (<-20 dBm), optic mismatch, or dirty connectors +**Solution:** +1. Check physical connections +2. Clean fiber connectors +3. Verify optic types (10GBASE-LR/100GBASE-LR4) +4. Test with known-good optics +5. Check patch panel +6. Contact account team + +### "BGP Session Down" + +**Cause:** Wrong IP addressing, wrong ASN, password mismatch, or firewall blocking TCP/179 +**Solution:** +1. Verify IPs match CNI object +2. Confirm ASN correct +3. Check BGP password +4. Verify no firewall on TCP/179 +5. Check BGP logs +6. Review BGP timers + +### "Low Throughput" + +**Cause:** MTU mismatch, fragmentation, single GRE tunnel (v1), or routing inefficiency +**Solution:** +1. Check MTU (1500↓/1476↑ for v1, 1500 both for v2) +2. Test various packet sizes +3. Add more GRE tunnels (v1) +4. Consider upgrading to v2 +5. Review routing tables +6. Use LACP for bundling (v1) + +## API Errors + +### 400 Bad Request: "slot_id already occupied" + +**Cause:** Another interconnect already uses this slot +**Solution:** Use `occupied=false` filter when listing slots: +```typescript +await client.networkInterconnects.slots.list({ + account_id: id, + occupied: false, + facility: 'EWR1', +}); +``` + +### 400 Bad Request: "invalid facility code" + +**Cause:** Typo or unsupported facility +**Solution:** Check [locations PDF](https://developers.cloudflare.com/network-interconnect/static/cni-locations-2026-01.pdf) for valid codes + +### 403 Forbidden: "Enterprise plan required" + +**Cause:** Account not enterprise-level +**Solution:** Contact account team to upgrade + +### 422 Unprocessable: "validate_only request failed" + +**Cause:** Dry-run validation found issues (wrong slot, invalid config) +**Solution:** Review error message details, fix config before real creation + +### Rate Limiting + +**Limit:** 1200 requests/5min per token +**Solution:** Implement exponential backoff, cache slot listings + +## Cloud-Specific Issues + +### AWS Direct Connect: "VLAN not matching" + +**Cause:** VLAN ID from AWS LOA doesn't match CNI config +**Solution:** +1. Get VLAN from AWS Console after ordering +2. Send exact VLAN to CF account team +3. Verify match in CNI object config + +### AWS: "Connection stuck in Pending" + +**Cause:** LOA not provided to CF or AWS connection not accepted +**Solution:** +1. Verify AWS connection status is "Available" +2. Confirm LOA sent to CF account team +3. Wait for CF team acceptance (can take days) + +### GCP: "BGP routes not propagating" + +**Cause:** BGP routes from GCP Cloud Router **ignored by design** +**Solution:** Use [static routes](https://developers.cloudflare.com/magic-wan/configuration/manually/how-to/configure-routes/#configure-static-routes) in Magic WAN instead + +### GCP: "Cannot query VLAN attachment status via API" + +**Cause:** GCP Cloud Interconnect Dashboard-only (no API yet) +**Solution:** Check status in CF Dashboard or GCP Console + +## Partner Interconnect Issues + +### Equinix: "Virtual circuit not appearing" + +**Cause:** CF hasn't accepted Equinix connection request +**Solution:** +1. Verify VC created in Equinix Fabric Portal +2. Contact CF account team to accept +3. Allow 2-3 business days + +### Console Connect/Megaport: "API creation fails" + +**Cause:** Partner interconnects require partner portal + CF approval +**Solution:** Cannot fully automate. Order in partner portal, notify CF account team. + +## Anti-Patterns + +| Anti-Pattern | Why Bad | Solution | +|--------------|---------|----------| +| Single interconnect for production | No SLA, single point of failure | Use ≥2 with device diversity | +| No backup Internet | CNI fails = total outage | Always maintain alternate path | +| Polling status every second | Rate limits, wastes API calls | Poll every 30-60s max | +| Using v1 for Magic WAN v2 workloads | GRE overhead, complexity | Use v2 for simplified routing | +| Assuming BGP session = traffic flowing | BGP up ≠ routes installed | Verify routing tables + test traffic | +| Not enabling maintenance alerts | Surprise downtime during maintenance | Enable notifications immediately | +| Hardcoding VLAN in automation | VLAN assigned by CF (v1) | Get VLAN from CNI object response | +| Using Direct without colocation | Can't access cross-connect | Use Partner or Cloud interconnect | + +## What's Not Queryable via API + +**Cannot retrieve:** +- BGP session state (use Dashboard or BGP logs) +- Light levels (contact account team) +- Historical metrics (uptime, traffic) +- Bandwidth utilization per interconnect +- Maintenance window schedules (notifications only) +- Fiber path details +- Cross-connect installation status + +**Workarounds:** +- External monitoring for BGP state +- Log aggregation for historical data +- Notifications for maintenance windows + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Max optical distance | 10km | Physical limit | +| MTU (v1) | 1500↓ / 1476↑ | Asymmetric | +| MTU (v2) | 1500 both | Symmetric | +| GRE tunnel throughput | 1 Gbps | Per tunnel (v1) | +| Recovery time | Days | No formal SLA | +| Light level minimum | -20 dBm | Target threshold | +| API rate limit | 1200 req/5min | Per token | +| Health check delay | 6 hours | New maintenance alert subscriptions | diff --git a/plugins/cloudflare/skills/cloudflare/references/network-interconnect/patterns.md b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/patterns.md new file mode 100644 index 0000000..7ff9dd3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/network-interconnect/patterns.md @@ -0,0 +1,166 @@ +# CNI Patterns + +See [README.md](README.md) for overview. + +## High Availability + +**Critical:** Design for resilience from day one. + +**Requirements:** +- Device-level diversity (separate hardware) +- Backup Internet connectivity (no SLA on CNI) +- Network-resilient locations preferred +- Regular failover testing + +**Architecture:** +``` +Your Network A ──10G CNI v2──> CF CCR Device 1 + │ +Your Network B ──10G CNI v2──> CF CCR Device 2 + │ + CF Global Network (AS13335) +``` + +**Capacity Planning:** +- Plan across all links +- Account for failover scenarios +- Your responsibility + +## Pattern: Magic Transit + CNI v2 + +**Use Case:** DDoS protection, private connectivity, no GRE overhead. + +```typescript +// 1. Create interconnect +const ic = await client.networkInterconnects.interconnects.create({ + account_id: id, + type: 'direct', + facility: 'EWR1', + speed: '10G', + name: 'magic-transit-primary', +}); + +// 2. Poll until active +const status = await pollUntilActive(id, ic.id); + +// 3. Configure Magic Transit tunnel via Dashboard/API +``` + +**Benefits:** 1500 MTU both ways, simplified routing. + +## Pattern: Multi-Cloud Hybrid + +**Use Case:** AWS/GCP workloads with Cloudflare. + +**AWS Direct Connect:** +```typescript +// 1. Order Direct Connect in AWS Console +// 2. Get LOA + VLAN from AWS +// 3. Send to CF account team (no API) +// 4. Configure static routes in Magic WAN + +await configureStaticRoutes(id, { + prefix: '10.0.0.0/8', + nexthop: 'aws-direct-connect', +}); +``` + +**GCP Cloud Interconnect:** +``` +1. Get VLAN attachment pairing key from GCP Console +2. Create via Dashboard: Interconnects → Create → Cloud Interconnect → Google + - Enter pairing key, name, MTU, speed +3. Configure static routes in Magic WAN (BGP routes from GCP ignored) +4. Configure custom learned routes in GCP Cloud Router +``` + +**Note:** Dashboard-only. No API/SDK support yet. + +## Pattern: Multi-Location HA + +**Use Case:** 99.99%+ uptime. + +```typescript +// Primary (NY) +const primary = await client.networkInterconnects.interconnects.create({ + account_id: id, + type: 'direct', + facility: 'EWR1', + speed: '10G', + name: 'primary-ewr1', +}); + +// Secondary (NY, different hardware) +const secondary = await client.networkInterconnects.interconnects.create({ + account_id: id, + type: 'direct', + facility: 'EWR2', + speed: '10G', + name: 'secondary-ewr2', +}); + +// Tertiary (LA, different geography) +const tertiary = await client.networkInterconnects.interconnects.create({ + account_id: id, + type: 'partner', + facility: 'LAX1', + speed: '10G', + name: 'tertiary-lax1', +}); + +// BGP local preferences: +// Primary: 200 +// Secondary: 150 +// Tertiary: 100 +// Internet: Last resort +``` + +## Pattern: Partner Interconnect (Equinix) + +**Use Case:** Quick deployment, no colocation. + +**Setup:** +1. Order virtual circuit in Equinix Fabric Portal +2. Select Cloudflare as destination +3. Choose facility +4. Send details to CF account team +5. CF accepts in portal +6. Configure BGP + +**No API automation** – partner portals managed separately. + +## Failover & Security + +**Failover Best Practices:** +- Use BGP local preferences for priority +- Configure BFD for fast detection (v1) +- Test regularly with traffic shift +- Document runbooks + +**Security:** +- BGP password authentication +- BGP route filtering +- Monitor unexpected routes +- Magic Firewall for DDoS/threats +- Minimum API token permissions +- Rotate credentials periodically + +## Decision Matrix + +| Requirement | Recommended | +|-------------|-------------| +| Collocated with CF | Direct | +| Not collocated | Partner | +| AWS/GCP workloads | Cloud | +| 1500 MTU both ways | v2 | +| VLAN tagging | v1 | +| Public peering | v1 | +| Simplest config | v2 | +| BFD fast failover | v1 | +| LACP bundling | v1 | + +## Resources + +- [Magic Transit Docs](https://developers.cloudflare.com/magic-transit/) +- [Magic WAN Docs](https://developers.cloudflare.com/magic-wan/) +- [Argo Smart Routing](https://developers.cloudflare.com/argo/) diff --git a/plugins/cloudflare/skills/cloudflare/references/observability/README.md b/plugins/cloudflare/skills/cloudflare/references/observability/README.md new file mode 100644 index 0000000..58feed6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/observability/README.md @@ -0,0 +1,87 @@ +# Cloudflare Observability Skill Reference + +**Purpose**: Comprehensive guidance for implementing observability in Cloudflare Workers, covering traces, logs, metrics, and analytics. + +**Scope**: Cloudflare Observability features ONLY - Workers Logs, Traces, Analytics Engine, Logpush, Metrics & Analytics, and OpenTelemetry exports. + +--- + +## Decision Tree: Which File to Load? + +Use this to route to the correct file without loading all content: + +``` +├─ "How do I enable/configure X?" → configuration.md +├─ "What's the API/method/binding for X?" → api.md +├─ "How do I implement X pattern?" → patterns.md +│ ├─ Usage tracking/billing → patterns.md +│ ├─ Error tracking → patterns.md +│ ├─ Performance monitoring → patterns.md +│ ├─ Multi-tenant tracking → patterns.md +│ ├─ Tail Worker filtering → patterns.md +│ └─ OpenTelemetry export → patterns.md +└─ "Why isn't X working?" / "Limits?" → gotchas.md +``` + +## Reading Order + +Load files in this order based on task: + +| Task Type | Load Order | Reason | +|-----------|------------|--------| +| **Initial setup** | configuration.md → gotchas.md | Setup first, avoid pitfalls | +| **Implement feature** | patterns.md → api.md → gotchas.md | Pattern → API details → edge cases | +| **Debug issue** | gotchas.md → configuration.md | Common issues first | +| **Query data** | api.md → patterns.md | API syntax → query examples | + +## Product Overview + +### Workers Logs +- **What:** Console output from Workers (console.log/warn/error) +- **Access:** Dashboard (Real-time Logs), Logpush, Tail Workers +- **Cost:** Free (included with all Workers) +- **Retention:** Real-time only (no historical storage in dashboard) + +### Workers Traces +- **What:** Execution traces with timing, CPU usage, outcome +- **Access:** Dashboard (Workers Analytics → Traces), Logpush +- **Cost:** $0.10/1M spans (GA pricing starts March 1, 2026), 10M free/month +- **Retention:** 14 days included + +### Analytics Engine +- **What:** High-cardinality event storage and SQL queries +- **Access:** SQL API, Dashboard (Analytics → Analytics Engine) +- **Cost:** $0.25/1M writes beyond 10M free/month +- **Retention:** 90 days (configurable up to 1 year) + +### Tail Workers +- **What:** Workers that receive logs/traces from other Workers +- **Use Cases:** Log filtering, transformation, external export +- **Cost:** Standard Workers pricing + +### Logpush +- **What:** Stream logs to external storage (S3, R2, Datadog, etc.) +- **Access:** Dashboard, API +- **Cost:** Requires Business/Enterprise plan + +## Pricing Summary (2026) + +| Feature | Free Tier | Cost Beyond Free Tier | Plan Requirement | +|---------|-----------|----------------------|------------------| +| Workers Logs | Unlimited | Free | Any | +| Workers Traces | 10M spans/month | $0.10/1M spans | Paid Workers (GA: March 1, 2026) | +| Analytics Engine | 10M writes/month | $0.25/1M writes | Paid Workers | +| Logpush | N/A | Included in plan | Business/Enterprise | + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, configuration (Logs, Traces, Analytics Engine, Tail Workers, Logpush) +- **[api.md](api.md)** - API endpoints, methods, interfaces (GraphQL, SQL, bindings, types) +- **[patterns.md](patterns.md)** - Common patterns, use cases, examples (billing, monitoring, error tracking, exports) +- **[gotchas.md](gotchas.md)** - Troubleshooting, best practices, limitations (common errors, performance gotchas, pricing) + +## See Also + +- [Cloudflare Workers Docs](https://developers.cloudflare.com/workers/) +- [Analytics Engine Docs](https://developers.cloudflare.com/analytics/analytics-engine/) +- [Workers Traces Docs](https://developers.cloudflare.com/workers/observability/traces/) diff --git a/plugins/cloudflare/skills/cloudflare/references/observability/api.md b/plugins/cloudflare/skills/cloudflare/references/observability/api.md new file mode 100644 index 0000000..a0161de --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/observability/api.md @@ -0,0 +1,164 @@ +## API Reference + +### GraphQL Analytics API + +**Endpoint**: `https://api.cloudflare.com/client/v4/graphql` + +**Query Workers Metrics**: +```graphql +query { + viewer { + accounts(filter: { accountTag: $accountId }) { + workersInvocationsAdaptive( + limit: 100 + filter: { + datetime_geq: "2025-01-01T00:00:00Z" + datetime_leq: "2025-01-31T23:59:59Z" + scriptName: "my-worker" + } + ) { + sum { + requests + errors + subrequests + } + quantiles { + cpuTimeP50 + cpuTimeP99 + wallTimeP50 + wallTimeP99 + } + } + } + } +} +``` + +### Analytics Engine SQL API + +**Endpoint**: `https://api.cloudflare.com/client/v4/accounts/{account_id}/analytics_engine/sql` + +**Authentication**: `Authorization: Bearer ` (Account Analytics Read permission) + +**Common Queries**: + +```sql +-- List all datasets +SHOW TABLES; + +-- Time-series aggregation (5-minute buckets) +SELECT + intDiv(toUInt32(timestamp), 300) * 300 AS time_bucket, + blob1 AS endpoint, + SUM(_sample_interval) AS total_requests, + AVG(double1) AS avg_response_time_ms +FROM api_metrics +WHERE timestamp >= NOW() - INTERVAL '24' HOUR +GROUP BY time_bucket, endpoint +ORDER BY time_bucket DESC; + +-- Top customers by usage +SELECT + index1 AS customer_id, + SUM(_sample_interval * double1) AS total_api_calls, + AVG(double2) AS avg_response_time_ms +FROM api_usage +WHERE timestamp >= NOW() - INTERVAL '7' DAY +GROUP BY customer_id +ORDER BY total_api_calls DESC +LIMIT 100; + +-- Error rate analysis +SELECT + blob1 AS error_type, + COUNT(*) AS occurrences, + MAX(timestamp) AS last_seen +FROM error_tracking +WHERE timestamp >= NOW() - INTERVAL '1' HOUR +GROUP BY error_type +ORDER BY occurrences DESC; +``` + +### Console Logging API + +**Methods**: +```typescript +// Standard methods (all appear in Workers Logs) +console.log('info message'); +console.info('info message'); +console.warn('warning message'); +console.error('error message'); +console.debug('debug message'); + +// Structured logging (recommended) +console.log({ + level: 'info', + user_id: '123', + action: 'checkout', + amount: 99.99, + currency: 'USD' +}); +``` + +**Log Levels**: All console methods produce logs; use structured fields for filtering: +```typescript +console.log({ + level: 'error', + message: 'Payment failed', + error_code: 'CARD_DECLINED' +}); +``` + +### Analytics Engine Binding Types + +```typescript +interface AnalyticsEngineDataset { + writeDataPoint(event: AnalyticsEngineDataPoint): void; +} + +interface AnalyticsEngineDataPoint { + // Indexed strings (use for filtering/grouping) + indexes?: string[]; + + // Non-indexed strings (metadata, IDs, URLs) + blobs?: string[]; + + // Numeric values (counts, durations, amounts) + doubles?: number[]; +} +``` + +**Field Limits**: +- Max 20 indexes +- Max 20 blobs +- Max 20 doubles +- Max 25 `writeDataPoint` calls per request + +### Tail Consumer Event Type + +```typescript +interface TraceItem { + event: TraceEvent; + logs: TraceLog[]; + exceptions: TraceException[]; + scriptName?: string; +} + +interface TraceEvent { + outcome: 'ok' | 'exception' | 'exceededCpu' | 'exceededMemory' | 'unknown'; + cpuTime: number; // microseconds + wallTime: number; // microseconds +} + +interface TraceLog { + timestamp: number; + level: 'log' | 'info' | 'debug' | 'warn' | 'error'; + message: any; // string or structured object +} + +interface TraceException { + name: string; + message: string; + timestamp: number; +} +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/observability/configuration.md b/plugins/cloudflare/skills/cloudflare/references/observability/configuration.md new file mode 100644 index 0000000..483de4c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/observability/configuration.md @@ -0,0 +1,169 @@ +## Configuration Patterns + +### Enable Workers Logs + +```jsonc +{ + "observability": { + "enabled": true, + "head_sampling_rate": 1 // 100% sampling (default) + } +} +``` + +**Best Practice**: Use structured JSON logging for better indexing + +```typescript +// Good - structured logging +console.log({ + user_id: 123, + action: "login", + status: "success", + duration_ms: 45 +}); + +// Avoid - unstructured string +console.log("user_id: 123 logged in successfully in 45ms"); +``` + +### Enable Workers Traces + +```jsonc +{ + "observability": { + "traces": { + "enabled": true, + "head_sampling_rate": 0.05 // 5% sampling + } + } +} +``` + +**Note**: Default sampling is 100%. For high-traffic Workers, use lower sampling (0.01-0.1). + +### Configure Analytics Engine + +**Bind to Worker**: +```toml +# wrangler.toml +analytics_engine_datasets = [ + { binding = "ANALYTICS", dataset = "api_metrics" } +] +``` + +**Write Data Points**: +```typescript +export interface Env { + ANALYTICS: AnalyticsEngineDataset; +} + +export default { + async fetch(request: Request, env: Env): Promise { + // Track metrics + env.ANALYTICS.writeDataPoint({ + blobs: ['customer_123', 'POST', '/api/v1/users'], + doubles: [1, 245.5], // request_count, response_time_ms + indexes: ['customer_123'] // for efficient filtering + }); + + return new Response('OK'); + } +} +``` + +### Configure Tail Workers + +Tail Workers receive logs/traces from other Workers for filtering, transformation, or export. + +**Setup**: +```toml +# wrangler.toml +name = "log-processor" +main = "src/tail.ts" + +[[tail_consumers]] +service = "my-worker" # Worker to tail +``` + +**Tail Worker Example**: +```typescript +export default { + async tail(events: TraceItem[], env: Env, ctx: ExecutionContext) { + // Filter errors only + const errors = events.filter(event => + event.outcome === 'exception' || event.outcome === 'exceededCpu' + ); + + if (errors.length > 0) { + // Send to external monitoring + ctx.waitUntil( + fetch('https://monitoring.example.com/errors', { + method: 'POST', + body: JSON.stringify(errors) + }) + ); + } + } +} +``` + +### Configure Logpush + +Send logs to external storage (S3, R2, GCS, Azure, Datadog, etc.). Requires Business/Enterprise plan. + +**Via Dashboard**: +1. Navigate to Analytics → Logs → Logpush +2. Select destination type +3. Provide credentials and bucket/endpoint +4. Choose dataset (e.g., Workers Trace Events) +5. Configure filters and fields + +**Via API**: +```bash +curl -X POST "https://api.cloudflare.com/client/v4/accounts/{account_id}/logpush/jobs" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "name": "workers-logs-to-s3", + "destination_conf": "s3://my-bucket/logs?region=us-east-1", + "dataset": "workers_trace_events", + "enabled": true, + "frequency": "high", + "filter": "{\"where\":{\"and\":[{\"key\":\"ScriptName\",\"operator\":\"eq\",\"value\":\"my-worker\"}]}}" + }' +``` + +### Environment-Specific Configuration + +**Development** (verbose logs, full sampling): +```jsonc +// wrangler.dev.jsonc +{ + "observability": { + "enabled": true, + "head_sampling_rate": 1.0, + "traces": { + "enabled": true + } + } +} +``` + +**Production** (reduced sampling, structured logs): +```jsonc +// wrangler.prod.jsonc +{ + "observability": { + "enabled": true, + "head_sampling_rate": 0.1, // 10% sampling + "traces": { + "enabled": true + } + } +} +``` + +Deploy with env-specific config: +```bash +wrangler deploy --config wrangler.prod.jsonc --env production +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/observability/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/observability/gotchas.md new file mode 100644 index 0000000..42bc738 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/observability/gotchas.md @@ -0,0 +1,115 @@ +## Common Errors + +### "Logs not appearing" + +**Cause:** Observability disabled, Worker not redeployed, no traffic, low sampling rate, or log size exceeds 256 KB +**Solution:** +```bash +# Verify config +cat wrangler.jsonc | jq '.observability' + +# Check deployment +wrangler deployments list + +# Test with curl +curl https://your-worker.workers.dev +``` +Ensure `observability.enabled = true`, redeploy Worker, check `head_sampling_rate`, verify traffic + +### "Traces not being captured" + +**Cause:** Traces not enabled, incorrect sampling rate, Worker not redeployed, or destination unavailable +**Solution:** +```jsonc +// Temporarily set to 100% sampling for debugging +{ + "observability": { + "enabled": true, + "head_sampling_rate": 1.0, + "traces": { + "enabled": true + } + } +} +``` +Ensure `observability.traces.enabled = true`, set `head_sampling_rate` to 1.0 for testing, redeploy, check destination status + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Max log size | 256 KB | Logs exceeding this are truncated | +| Default sampling rate | 1.0 (100%) | Reduce for high-traffic Workers | +| Max destinations | Varies by plan | Check dashboard | +| Trace context propagation | 100 spans max | Deep call chains may lose spans | +| Analytics Engine write rate | 25 writes/request | Excess writes dropped silently | + +## Performance Gotchas + +### Spectre Mitigation Timing + +**Problem:** `Date.now()` and `performance.now()` have reduced precision (coarsened to 100μs) +**Cause:** Spectre vulnerability mitigation in V8 +**Solution:** Accept reduced precision or use Workers Traces for accurate timing +```typescript +// Date.now() is coarsened - trace spans are accurate +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + // For user-facing timing, Date.now() is fine + const start = Date.now(); + const response = await processRequest(request); + const duration = Date.now() - start; + + // For detailed performance analysis, use Workers Traces instead + return response; + } +} +``` + +### Analytics Engine _sample_interval Aggregation + +**Problem:** Queries return incorrect totals when not multiplying by `_sample_interval` +**Cause:** Analytics Engine stores sampled data points, each representing multiple events +**Solution:** Always multiply counts/sums by `_sample_interval` in aggregations +```sql +-- WRONG: Undercounts actual events +SELECT blob1 AS customer_id, COUNT(*) AS total_calls +FROM api_usage GROUP BY customer_id; + +-- CORRECT: Accounts for sampling +SELECT blob1 AS customer_id, SUM(_sample_interval) AS total_calls +FROM api_usage GROUP BY customer_id; +``` + +### Trace Context Propagation Limits + +**Problem:** Deep call chains lose trace context after 100 spans +**Cause:** Cloudflare limits trace depth to prevent performance impact +**Solution:** Design for flatter architectures or use custom correlation IDs for deep chains +```typescript +// For deep call chains, add custom correlation ID +const correlationId = crypto.randomUUID(); +console.log({ correlationId, event: 'request_start' }); + +// Pass correlationId through headers to downstream services +await fetch('https://api.example.com', { + headers: { 'X-Correlation-ID': correlationId } +}); +``` + +## Pricing (2026) + +### Workers Traces +- **GA Pricing (starts March 1, 2026):** + - $0.10 per 1M trace spans captured + - Retention: 14 days included +- **Free tier:** 10M trace spans/month +- **Note:** Beta usage (before March 1, 2026) is free + +### Workers Logs +- **Included:** Free for all Workers +- **Logpush:** Requires Business/Enterprise plan + +### Analytics Engine +- **Included:** 10M writes/month on Paid Workers plan +- **Additional:** $0.25 per 1M writes beyond included quota diff --git a/plugins/cloudflare/skills/cloudflare/references/observability/patterns.md b/plugins/cloudflare/skills/cloudflare/references/observability/patterns.md new file mode 100644 index 0000000..9135c68 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/observability/patterns.md @@ -0,0 +1,105 @@ +# Observability Patterns + +## Usage-Based Billing + +```typescript +env.ANALYTICS.writeDataPoint({ + blobs: [customerId, request.url, request.method], + doubles: [1], // request_count + indexes: [customerId] +}); +``` + +```sql +SELECT blob1 AS customer_id, SUM(_sample_interval * double1) AS total_calls +FROM api_usage WHERE timestamp >= DATE_TRUNC('month', NOW()) +GROUP BY customer_id +``` + +## Performance Monitoring + +```typescript +const start = Date.now(); +const response = await fetch(url); +env.ANALYTICS.writeDataPoint({ + blobs: [url, response.status.toString()], + doubles: [Date.now() - start, response.status] +}); +``` + +```sql +SELECT blob1 AS url, AVG(double1) AS avg_ms, percentile(double1, 0.95) AS p95_ms +FROM fetch_metrics WHERE timestamp >= NOW() - INTERVAL '1' HOUR +GROUP BY url +``` + +## Error Tracking + +```typescript +env.ANALYTICS.writeDataPoint({ + blobs: [error.name, request.url, request.method], + doubles: [1], + indexes: [error.name] +}); +``` + +## Multi-Tenant Tracking + +```typescript +env.ANALYTICS.writeDataPoint({ + indexes: [tenantId], // efficient filtering + blobs: [tenantId, url.pathname, method, status], + doubles: [1, duration, bytesSize] +}); +``` + +## Tail Worker Log Filtering + +```typescript +export default { + async tail(events, env, ctx) { + const critical = events.filter(e => + e.exceptions.length > 0 || e.event.wallTime > 1000000 + ); + if (critical.length === 0) return; + + ctx.waitUntil( + fetch('https://logging.example.com/ingest', { + method: 'POST', + headers: { 'Authorization': `Bearer ${env.API_KEY}` }, + body: JSON.stringify(critical.map(e => ({ + outcome: e.event.outcome, + cpu_ms: e.event.cpuTime / 1000, + errors: e.exceptions + }))) + }) + ); + } +}; +``` + +## OpenTelemetry Export + +```typescript +export default { + async tail(events, env, ctx) { + const otelSpans = events.map(e => ({ + traceId: generateId(32), + spanId: generateId(16), + name: e.scriptName || 'worker.request', + attributes: [ + { key: 'worker.outcome', value: { stringValue: e.event.outcome } }, + { key: 'worker.cpu_time_us', value: { intValue: String(e.event.cpuTime) } } + ] + })); + + ctx.waitUntil( + fetch('https://api.honeycomb.io/v1/traces', { + method: 'POST', + headers: { 'X-Honeycomb-Team': env.HONEYCOMB_KEY }, + body: JSON.stringify({ resourceSpans: [{ scopeSpans: [{ spans: otelSpans }] }] }) + }) + ); + } +}; +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/pages-functions/README.md b/plugins/cloudflare/skills/cloudflare/references/pages-functions/README.md new file mode 100644 index 0000000..deaf461 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages-functions/README.md @@ -0,0 +1,98 @@ +# Cloudflare Pages Functions + +Serverless functions on Cloudflare Pages using Workers runtime. Full-stack dev with file-based routing. + +## Quick Navigation + +**Need to...** +| Task | Go to | +|------|-------| +| Set up TypeScript types | [configuration.md](./configuration.md) - TypeScript Setup | +| Configure bindings (KV, D1, R2) | [configuration.md](./configuration.md) - wrangler.jsonc | +| Access request/env/params | [api.md](./api.md) - EventContext | +| Add middleware or auth | [patterns.md](./patterns.md) - Middleware, Auth | +| Background tasks (waitUntil) | [patterns.md](./patterns.md) - Background Tasks | +| Debug errors or check limits | [gotchas.md](./gotchas.md) - Common Errors, Limits | + +## Decision Tree: Is This Pages Functions? + +``` +Need serverless backend? +├─ Yes, for a static site → Pages Functions +├─ Yes, standalone API → Workers +└─ Just static hosting → Pages (no functions) + +Have existing Worker? +├─ Complex routing logic → Use _worker.js (Advanced Mode) +└─ Simple routes → Migrate to /functions (File-Based) + +Framework-based? +├─ Next.js/SvelteKit/Remix → Uses _worker.js automatically +└─ Vanilla/HTML/React SPA → Use /functions +``` + +## File-Based Routing + +``` +/functions + ├── index.js → / + ├── api.js → /api + ├── users/ + │ ├── index.js → /users/ + │ ├── [user].js → /users/:user + │ └── [[catchall]].js → /users/* + └── _middleware.js → runs on all routes +``` + +**Rules:** +- `index.js` → directory root +- Trailing slash optional +- Specific routes precede catch-alls +- Falls back to static if no match + +## Dynamic Routes + +**Single segment** `[param]` → string: +```js +// /functions/users/[user].js +export function onRequest(context) { + return new Response(`Hello ${context.params.user}`); +} +// Matches: /users/nevi +``` + +**Multi-segment** `[[param]]` → array: +```js +// /functions/users/[[catchall]].js +export function onRequest(context) { + return new Response(JSON.stringify(context.params.catchall)); +} +// Matches: /users/nevi/foobar → ["nevi", "foobar"] +``` + +## Key Features + +- **Method handlers:** `onRequestGet`, `onRequestPost`, etc. +- **Middleware:** `_middleware.js` for cross-cutting concerns +- **Bindings:** KV, D1, R2, Durable Objects, Workers AI, Service bindings +- **TypeScript:** Full type support via `wrangler types` command +- **Advanced mode:** Use `_worker.js` for custom routing logic + +## Reading Order + +**New to Pages Functions?** Start here: +1. [README.md](./README.md) - Overview, routing, decision tree (you are here) +2. [configuration.md](./configuration.md) - TypeScript setup, wrangler.jsonc, bindings +3. [api.md](./api.md) - EventContext, handlers, bindings reference +4. [patterns.md](./patterns.md) - Middleware, auth, CORS, rate limiting, caching +5. [gotchas.md](./gotchas.md) - Common errors, debugging, limits + +**Quick reference lookup:** +- Bindings table → [api.md](./api.md) +- Error diagnosis → [gotchas.md](./gotchas.md) +- TypeScript setup → [configuration.md](./configuration.md) + +## See Also +- [pages](../pages/) - Pages platform overview and static site deployment +- [workers](../workers/) - Workers runtime API reference +- [d1](../d1/) - D1 database integration with Pages Functions diff --git a/plugins/cloudflare/skills/cloudflare/references/pages-functions/api.md b/plugins/cloudflare/skills/cloudflare/references/pages-functions/api.md new file mode 100644 index 0000000..5263372 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages-functions/api.md @@ -0,0 +1,143 @@ +# Function API + +## EventContext + +```typescript +interface EventContext { + request: Request; // Incoming request + functionPath: string; // Request path + waitUntil(promise: Promise): void; // Background tasks (non-blocking) + passThroughOnException(): void; // Fallback to static on error + next(input?: Request | string, init?: RequestInit): Promise; + env: Env; // Bindings, vars, secrets + params: Record; // Route params ([user] or [[catchall]]) + data: any; // Middleware shared state +} +``` + +**TypeScript:** See [configuration.md](./configuration.md) for `wrangler types` setup + +## Handlers + +```typescript +// Generic (fallback for any method) +export async function onRequest(ctx: EventContext): Promise { + return new Response('Any method'); +} + +// Method-specific (takes precedence over generic) +export async function onRequestGet(ctx: EventContext): Promise { + return Response.json({ message: 'GET' }); +} + +export async function onRequestPost(ctx: EventContext): Promise { + const body = await ctx.request.json(); + return Response.json({ received: body }); +} +// Also: onRequestPut, onRequestPatch, onRequestDelete, onRequestHead, onRequestOptions +``` + +## Bindings Reference + +| Binding Type | Interface | Config Key | Use Case | +|--------------|-----------|------------|----------| +| KV | `KVNamespace` | `kv_namespaces` | Key-value cache, sessions, config | +| D1 | `D1Database` | `d1_databases` | Relational data, SQL queries | +| R2 | `R2Bucket` | `r2_buckets` | Large files, user uploads, assets | +| Durable Objects | `DurableObjectNamespace` | `durable_objects.bindings` | Stateful coordination, websockets | +| Workers AI | `Ai` | `ai.binding` | LLM inference, embeddings | +| Vectorize | `VectorizeIndex` | `vectorize` | Vector search, embeddings | +| Service Binding | `Fetcher` | `services` | Worker-to-worker RPC | +| Analytics Engine | `AnalyticsEngineDataset` | `analytics_engine_datasets` | Event logging, metrics | +| Environment Vars | `string` | `vars` | Non-sensitive config | + +See [configuration.md](./configuration.md) for wrangler.jsonc examples. + +## Bindings + +### KV + +```typescript +interface Env { KV: KVNamespace; } +export const onRequest: PagesFunction = async (ctx) => { + await ctx.env.KV.put('key', 'value', { expirationTtl: 3600 }); + const val = await ctx.env.KV.get('key', { type: 'json' }); + const keys = await ctx.env.KV.list({ prefix: 'user:' }); + return Response.json({ val }); +}; +``` + +### D1 + +```typescript +interface Env { DB: D1Database; } +export const onRequest: PagesFunction = async (ctx) => { + const user = await ctx.env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(123).first(); + return Response.json(user); +}; +``` + +### R2 + +```typescript +interface Env { BUCKET: R2Bucket; } +export const onRequest: PagesFunction = async (ctx) => { + const obj = await ctx.env.BUCKET.get('file.txt'); + if (!obj) return new Response('Not found', { status: 404 }); + await ctx.env.BUCKET.put('file.txt', ctx.request.body); + return new Response(obj.body); +}; +``` + +### Durable Objects + +```typescript +interface Env { COUNTER: DurableObjectNamespace; } +export const onRequest: PagesFunction = async (ctx) => { + const stub = ctx.env.COUNTER.get(ctx.env.COUNTER.idFromName('global')); + return stub.fetch(ctx.request); +}; +``` + +### Workers AI + +```typescript +interface Env { AI: Ai; } +export const onRequest: PagesFunction = async (ctx) => { + const resp = await ctx.env.AI.run('@cf/meta/llama-3.1-8b-instruct', { prompt: 'Hello' }); + return Response.json(resp); +}; +``` + +### Service Bindings & Env Vars + +```typescript +interface Env { AUTH: Fetcher; API_KEY: string; } +export const onRequest: PagesFunction = async (ctx) => { + // Service binding: forward to another Worker + return ctx.env.AUTH.fetch(ctx.request); + + // Environment variable + return Response.json({ key: ctx.env.API_KEY }); +}; +``` + +## Advanced Mode (env.ASSETS) + +When using `_worker.js`, access static assets via `env.ASSETS.fetch()`: + +```typescript +interface Env { ASSETS: Fetcher; KV: KVNamespace; } + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname.startsWith('/api/')) { + return Response.json({ data: await env.KV.get('key') }); + } + return env.ASSETS.fetch(request); // Fallback to static + } +} satisfies ExportedHandler; +``` + +**See also:** [configuration.md](./configuration.md) for TypeScript setup and wrangler.jsonc | [patterns.md](./patterns.md) for middleware and auth patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/pages-functions/configuration.md b/plugins/cloudflare/skills/cloudflare/references/pages-functions/configuration.md new file mode 100644 index 0000000..62ba298 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages-functions/configuration.md @@ -0,0 +1,122 @@ +# Configuration + +## TypeScript Setup + +**Generate types from wrangler.jsonc** (replaces deprecated `@cloudflare/workers-types`): + +```bash +npx wrangler types +``` + +Creates `worker-configuration.d.ts` with typed `Env` interface based on your bindings. + +```typescript +// functions/api.ts +export const onRequest: PagesFunction = async (ctx) => { + // ctx.env.KV, ctx.env.DB, etc. are fully typed + return Response.json({ ok: true }); +}; +``` + +**Manual types** (if not using wrangler types): + +```typescript +interface Env { + KV: KVNamespace; + DB: D1Database; + API_KEY: string; +} +export const onRequest: PagesFunction = async (ctx) => { /* ... */ }; +``` + +## wrangler.jsonc + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-pages-app", + "pages_build_output_dir": "./dist", + "compatibility_date": "2025-01-01", + "compatibility_flags": ["nodejs_compat"], + + "vars": { "API_URL": "https://api.example.com" }, + "kv_namespaces": [{ "binding": "KV", "id": "abc123" }], + "d1_databases": [{ "binding": "DB", "database_name": "prod-db", "database_id": "xyz789" }], + "r2_buckets": [{ "binding": "BUCKET", "bucket_name": "my-bucket" }], + "durable_objects": { "bindings": [{ "name": "COUNTER", "class_name": "Counter", "script_name": "counter-worker" }] }, + "services": [{ "binding": "AUTH", "service": "auth-worker" }], + "ai": { "binding": "AI" }, + "vectorize": [{ "binding": "VECTORIZE", "index_name": "my-index" }], + "analytics_engine_datasets": [{ "binding": "ANALYTICS" }] +} +``` + +## Environment Overrides + +Top-level → local dev, `env.preview` → preview, `env.production` → production + +```jsonc +{ + "vars": { "API_URL": "http://localhost:8787" }, + "env": { + "production": { "vars": { "API_URL": "https://api.example.com" } } + } +} +``` + +**Note:** If overriding `vars`, `kv_namespaces`, `d1_databases`, etc., ALL must be redefined (non-inheritable) + +## Local Secrets (.dev.vars) + +**Local dev only** - NOT deployed: + +```bash +# .dev.vars (add to .gitignore) +SECRET_KEY="my-secret-value" +``` + +Accessed via `ctx.env.SECRET_KEY`. Set production secrets: +```bash +echo "value" | npx wrangler pages secret put SECRET_KEY --project-name=my-app +``` + +## Static Config Files + +**_routes.json** - Custom routing: +```json +{ "version": 1, "include": ["/api/*"], "exclude": ["/static/*"] } +``` + +**_headers** - Static headers: +``` +/static/* + Cache-Control: public, max-age=31536000 +``` + +**_redirects** - Redirects: +``` +/old /new 301 +``` + +## Local Dev & Deployment + +```bash +# Dev server +npx wrangler pages dev ./dist + +# With bindings +npx wrangler pages dev ./dist --kv=KV --d1=DB=db-id --r2=BUCKET + +# Durable Objects (2 terminals) +cd do-worker && npx wrangler dev +cd pages-project && npx wrangler pages dev ./dist --do COUNTER=Counter@do-worker + +# Deploy +npx wrangler pages deploy ./dist +npx wrangler pages deploy ./dist --branch preview + +# Download config +npx wrangler pages download config my-project +``` + +**See also:** [api.md](./api.md) for binding usage examples diff --git a/plugins/cloudflare/skills/cloudflare/references/pages-functions/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/pages-functions/gotchas.md new file mode 100644 index 0000000..f63e608 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages-functions/gotchas.md @@ -0,0 +1,94 @@ +# Gotchas & Debugging + +## Error Diagnosis + +| Symptom | Likely Cause | Solution | +|---------|--------------|----------| +| **Function not invoking** | Wrong `/functions` location, wrong extension, or `_routes.json` excludes path | Check `pages_build_output_dir`, use `.js`/`.ts`, verify `_routes.json` | +| **`ctx.env.BINDING` undefined** | Binding not configured or name mismatch | Add to `wrangler.jsonc`, verify exact name (case-sensitive), redeploy | +| **TypeScript errors on `ctx.env`** | Missing type definition | Run `wrangler types` or define `interface Env {}` | +| **Middleware not running** | Wrong filename/location or missing `ctx.next()` | Name exactly `_middleware.js`, export `onRequest`, call `ctx.next()` | +| **Secrets missing in production** | `.dev.vars` not deployed | `.dev.vars` is local only - set production secrets via dashboard or `wrangler secret put` | +| **Type mismatch on binding** | Wrong interface type | See [api.md](./api.md) bindings table for correct types | +| **"KV key not found" but exists** | Key in wrong namespace or env | Verify namespace binding, check preview vs production env | +| **Function times out** | Synchronous wait or missing `await` | All I/O must be async/await, use `ctx.waitUntil()` for background tasks | + +## Common Errors + +### TypeScript type errors + +**Problem:** `ctx.env.MY_BINDING` shows type error +**Cause:** No type definition for `Env` +**Solution:** Run `npx wrangler types` or manually define: +```typescript +interface Env { MY_BINDING: KVNamespace; } +export const onRequest: PagesFunction = async (ctx) => { /* ... */ }; +``` + +### Secrets not available in production + +**Problem:** `ctx.env.SECRET_KEY` is undefined in production +**Cause:** `.dev.vars` is local-only, not deployed +**Solution:** Set production secrets: +```bash +echo "value" | npx wrangler pages secret put SECRET_KEY --project-name=my-app +``` + +## Debugging + +```typescript +// Console logging +export async function onRequest(ctx) { + console.log('Request:', ctx.request.method, ctx.request.url); + const res = await ctx.next(); + console.log('Status:', res.status); + return res; +} +``` + +```bash +# Stream real-time logs +npx wrangler pages deployment tail +npx wrangler pages deployment tail --status error +``` + +```jsonc +// Source maps (wrangler.jsonc) +{ "upload_source_maps": true } +``` + +## Limits + +| Resource | Free | Paid | +|----------|------|------| +| CPU time | 10ms | 50ms | +| Memory | 128 MB | 128 MB | +| Script size | 10 MB compressed | 10 MB compressed | +| Env vars | 5 KB per var, 64 max | 5 KB per var, 64 max | +| Requests | 100k/day | Unlimited ($0.50/million) | + +## Best Practices + +**Performance:** Minimize deps (cold start), use KV for cache/D1 for relational/R2 for large files, set `Cache-Control` headers, batch DB ops, handle errors gracefully + +**Security:** Never commit secrets (use `.dev.vars` + gitignore), validate input, sanitize before DB, implement auth middleware, set CORS headers, rate limit per-IP + +## Migration + +**Workers → Pages Functions:** +- `export default { fetch(req, env) {} }` → `export function onRequest(ctx) { const { request, env } = ctx; }` +- Use `_worker.js` for complex routing: `env.ASSETS.fetch(request)` for static files + +**Other platforms → Pages:** +- File-based routing: `/functions/api/users.js` → `/api/users` +- Dynamic routes: `[param]` not `:param` +- Replace Node.js deps with Workers APIs or add `nodejs_compat` flag + +## Resources + +- [Official Docs](https://developers.cloudflare.com/pages/functions/) +- [Workers APIs](https://developers.cloudflare.com/workers/runtime-apis/) +- [Examples](https://github.com/cloudflare/pages-example-projects) +- [Discord](https://discord.gg/cloudflaredev) + +**See also:** [configuration.md](./configuration.md) for TypeScript setup | [patterns.md](./patterns.md) for middleware/auth | [api.md](./api.md) for bindings diff --git a/plugins/cloudflare/skills/cloudflare/references/pages-functions/patterns.md b/plugins/cloudflare/skills/cloudflare/references/pages-functions/patterns.md new file mode 100644 index 0000000..22289e8 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages-functions/patterns.md @@ -0,0 +1,137 @@ +# Common Patterns + +## Background Tasks (waitUntil) + +Non-blocking tasks after response sent (analytics, cleanup, webhooks): + +```typescript +export async function onRequest(ctx: EventContext) { + const res = Response.json({ success: true }); + + ctx.waitUntil(ctx.env.KV.put('last-visit', new Date().toISOString())); + ctx.waitUntil(Promise.all([ + ctx.env.ANALYTICS.writeDataPoint({ event: 'view' }), + fetch('https://webhook.site/...', { method: 'POST' }) + ])); + + return res; // Returned immediately +} +``` + +## Middleware & Auth + +```typescript +// functions/_middleware.js (global) or functions/users/_middleware.js (scoped) +export async function onRequest(ctx) { + try { return await ctx.next(); } + catch (err) { return new Response(err.message, { status: 500 }); } +} + +// Chained: export const onRequest = [errorHandler, auth, logger]; + +// Auth +async function auth(ctx: EventContext) { + const token = ctx.request.headers.get('authorization')?.replace('Bearer ', ''); + if (!token) return new Response('Unauthorized', { status: 401 }); + const session = await ctx.env.KV.get(`session:${token}`); + if (!session) return new Response('Invalid', { status: 401 }); + ctx.data.user = JSON.parse(session); + return ctx.next(); +} +``` + +## CORS & Rate Limiting + +```typescript +// CORS middleware +const cors = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, POST' }; +export async function onRequestOptions() { return new Response(null, { headers: cors }); } +export async function onRequest(ctx) { + const res = await ctx.next(); + Object.entries(cors).forEach(([k, v]) => res.headers.set(k, v)); + return res; +} + +// Rate limiting (KV-based) +async function rateLimit(ctx: EventContext) { + const ip = ctx.request.headers.get('CF-Connecting-IP') || 'unknown'; + const count = parseInt(await ctx.env.KV.get(`rate:${ip}`) || '0'); + if (count >= 100) return new Response('Rate limited', { status: 429 }); + await ctx.env.KV.put(`rate:${ip}`, (count + 1).toString(), { expirationTtl: 3600 }); + return ctx.next(); +} +``` + +## Forms, Caching, Redirects + +```typescript +// JSON & file upload +export async function onRequestPost(ctx) { + const ct = ctx.request.headers.get('content-type') || ''; + if (ct.includes('application/json')) return Response.json(await ctx.request.json()); + if (ct.includes('multipart/form-data')) { + const file = (await ctx.request.formData()).get('file') as File; + await ctx.env.BUCKET.put(file.name, file.stream()); + return Response.json({ uploaded: file.name }); + } +} + +// Cache API +export async function onRequest(ctx) { + let res = await caches.default.match(ctx.request); + if (!res) { + res = new Response('Data'); + res.headers.set('Cache-Control', 'public, max-age=3600'); + ctx.waitUntil(caches.default.put(ctx.request, res.clone())); + } + return res; +} + +// Redirects +export async function onRequest(ctx) { + if (new URL(ctx.request.url).pathname === '/old') { + return Response.redirect(new URL('/new', ctx.request.url), 301); + } + return ctx.next(); +} +``` + +## Testing + +**Unit tests** (Vitest + cloudflare:test): +```typescript +import { env } from 'cloudflare:test'; +import { it, expect } from 'vitest'; +import { onRequest } from '../functions/api'; + +it('returns JSON', async () => { + const req = new Request('http://localhost/api'); + const ctx = { request: req, env, params: {}, data: {} } as EventContext; + const res = await onRequest(ctx); + expect(res.status).toBe(200); +}); +``` + +**Integration:** `wrangler pages dev` + Playwright/Cypress + +## Advanced Mode (_worker.js) + +Use `_worker.js` for complex routing (replaces `/functions`): + +```typescript +interface Env { ASSETS: Fetcher; KV: KVNamespace; } + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname.startsWith('/api/')) { + return Response.json({ data: await env.KV.get('key') }); + } + return env.ASSETS.fetch(request); // Static files + } +} satisfies ExportedHandler; +``` + +**When:** Existing Worker, framework-generated (Next.js/SvelteKit), custom routing logic + +**See also:** [api.md](./api.md) for `env.ASSETS.fetch()` | [gotchas.md](./gotchas.md) for debugging diff --git a/plugins/cloudflare/skills/cloudflare/references/pages/README.md b/plugins/cloudflare/skills/cloudflare/references/pages/README.md new file mode 100644 index 0000000..bf0546f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages/README.md @@ -0,0 +1,88 @@ +# Cloudflare Pages + +JAMstack platform for full-stack apps on Cloudflare's global network. + +## Key Features + +- **Git-based deploys**: Auto-deploy from GitHub/GitLab +- **Preview deployments**: Unique URL per branch/PR +- **Pages Functions**: File-based serverless routing (Workers runtime) +- **Static + dynamic**: Smart asset caching + edge compute +- **Smart Placement**: Automatic function optimization based on traffic patterns +- **Framework optimized**: SvelteKit, Astro, Nuxt, Qwik, Solid Start + +## Deployment Methods + +### 1. Git Integration (Production) +Dashboard → Workers & Pages → Create → Connect to Git → Configure build + +### 2. Direct Upload +```bash +npx wrangler pages deploy ./dist --project-name=my-project +npx wrangler pages deploy ./dist --project-name=my-project --branch=staging +``` + +### 3. C3 CLI +```bash +npm create cloudflare@latest my-app +# Select framework → auto-setup + deploy +``` + +## vs Workers + +- **Pages**: Static sites, JAMstack, frameworks, git workflow, file-based routing +- **Workers**: Pure APIs, complex routing, WebSockets, scheduled tasks, email handlers +- **Combine**: Pages Functions use Workers runtime, can bind to Workers + +## Quick Start + +```bash +# Create +npm create cloudflare@latest + +# Local dev +npx wrangler pages dev ./dist + +# Deploy +npx wrangler pages deploy ./dist --project-name=my-project + +# Types +npx wrangler types --path='./functions/types.d.ts' + +# Secrets +echo "value" | npx wrangler pages secret put KEY --project-name=my-project + +# Logs +npx wrangler pages deployment tail --project-name=my-project +``` + +## Resources + +- [Pages Docs](https://developers.cloudflare.com/pages/) +- [Functions API](https://developers.cloudflare.com/pages/functions/api-reference/) +- [Framework Guides](https://developers.cloudflare.com/pages/framework-guides/) +- [Discord #functions](https://discord.com/channels/595317990191398933/910978223968518144) + +## Reading Order + +**New to Pages?** Start here: +1. README.md (you are here) - Overview & quick start +2. [configuration.md](./configuration.md) - Project setup, wrangler.jsonc, bindings +3. [api.md](./api.md) - Functions API, routing, context +4. [patterns.md](./patterns.md) - Common implementations +5. [gotchas.md](./gotchas.md) - Troubleshooting & pitfalls + +**Quick reference?** Jump to relevant file above. + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc, build, env vars, Smart Placement +- [api.md](./api.md) - Functions API, bindings, context, advanced mode +- [patterns.md](./patterns.md) - Full-stack patterns, framework integration +- [gotchas.md](./gotchas.md) - Build issues, limits, debugging, framework warnings + +## See Also + +- [pages-functions](../pages-functions/) - File-based routing, middleware +- [d1](../d1/) - SQL database for Pages Functions +- [kv](../kv/) - Key-value storage for caching/state diff --git a/plugins/cloudflare/skills/cloudflare/references/pages/api.md b/plugins/cloudflare/skills/cloudflare/references/pages/api.md new file mode 100644 index 0000000..a719585 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages/api.md @@ -0,0 +1,204 @@ +# Functions API + +## File-Based Routing + +``` +/functions/index.ts → example.com/ +/functions/api/users.ts → example.com/api/users +/functions/api/users/[id].ts → example.com/api/users/:id +/functions/api/users/[[path]].ts → example.com/api/users/* (catchall) +/functions/_middleware.ts → Runs before all routes +``` + +**Rules**: `[param]` = single segment, `[[param]]` = multi-segment catchall, more specific wins. + +## Request Handlers + +```typescript +import type { PagesFunction } from '@cloudflare/workers-types'; + +interface Env { + DB: D1Database; + KV: KVNamespace; +} + +// All methods +export const onRequest: PagesFunction = async (context) => { + return new Response('All methods'); +}; + +// Method-specific +export const onRequestGet: PagesFunction = async (context) => { + const { request, env, params, data } = context; + + const user = await env.DB.prepare( + 'SELECT * FROM users WHERE id = ?' + ).bind(params.id).first(); + + return Response.json(user); +}; + +export const onRequestPost: PagesFunction = async (context) => { + const body = await context.request.json(); + return Response.json({ success: true }); +}; + +// Also: onRequestPut, onRequestPatch, onRequestDelete, onRequestHead, onRequestOptions +``` + +## Context Object + +```typescript +interface EventContext { + request: Request; // HTTP request + env: Env; // Bindings (KV, D1, R2, etc.) + params: Params; // Route parameters + data: Data; // Middleware-shared data + waitUntil: (promise: Promise) => void; // Background tasks + next: () => Promise; // Next handler + passThroughOnException: () => void; // Error fallback (not in advanced mode) +} +``` + +## Dynamic Routes + +```typescript +// Single segment: functions/users/[id].ts +export const onRequestGet: PagesFunction = async ({ params }) => { + // /users/123 → params.id = "123" + return Response.json({ userId: params.id }); +}; + +// Multi-segment: functions/files/[[path]].ts +export const onRequestGet: PagesFunction = async ({ params }) => { + // /files/docs/api/v1.md → params.path = ["docs", "api", "v1.md"] + const filePath = (params.path as string[]).join('/'); + return new Response(filePath); +}; +``` + +## Middleware + +```typescript +// functions/_middleware.ts +// Single +export const onRequest: PagesFunction = async (context) => { + const response = await context.next(); + response.headers.set('X-Custom-Header', 'value'); + return response; +}; + +// Chained (runs in order) +const errorHandler: PagesFunction = async (context) => { + try { + return await context.next(); + } catch (err) { + return new Response(err.message, { status: 500 }); + } +}; + +const auth: PagesFunction = async (context) => { + const token = context.request.headers.get('Authorization'); + if (!token) return new Response('Unauthorized', { status: 401 }); + context.data.userId = await verifyToken(token); + return context.next(); +}; + +export const onRequest = [errorHandler, auth]; +``` + +**Scope**: `functions/_middleware.ts` → all; `functions/api/_middleware.ts` → `/api/*` only + +## Bindings Usage + +```typescript +export const onRequestGet: PagesFunction = async ({ env }) => { + // KV + const cached = await env.KV.get('key', 'json'); + await env.KV.put('key', JSON.stringify({data: 'value'}), {expirationTtl: 3600}); + + // D1 + const result = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); + + // R2, Queue, AI - see respective reference docs + + return Response.json({success: true}); +}; +``` + +## Advanced Mode + +Full Workers API, bypasses file-based routing: + +```javascript +// functions/_worker.js +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + + // Custom routing + if (url.pathname.startsWith('/api/')) { + return new Response('API response'); + } + + // REQUIRED: Serve static assets + return env.ASSETS.fetch(request); + } +}; +``` + +**When to use**: WebSockets, complex routing, scheduled handlers, email handlers. + +## Smart Placement + +Automatically optimizes function execution location based on traffic patterns. + +**Configuration** (in wrangler.jsonc): +```jsonc +{ + "placement": { + "mode": "smart" // Enables optimization (default: off) + } +} +``` + +**How it works**: Analyzes traffic patterns over time and places functions closer to users or data sources (e.g., D1 databases). Requires no code changes. + +**Trade-offs**: Initial requests may see slightly higher latency during learning period (hours-days). Performance improves as system optimizes. + +**When to use**: Global apps with centralized databases or geographically concentrated traffic sources. + +## getRequestContext (Framework SSR) + +Access bindings in framework code: + +```typescript +// SvelteKit +import type { RequestEvent } from '@sveltejs/kit'; +export async function load({ platform }: RequestEvent) { + const data = await platform.env.DB.prepare('SELECT * FROM users').all(); + return { users: data.results }; +} + +// Astro +const { DB } = Astro.locals.runtime.env; +const data = await DB.prepare('SELECT * FROM users').all(); + +// Solid Start (server function) +import { getRequestEvent } from 'solid-js/web'; +const event = getRequestEvent(); +const data = await event.locals.runtime.env.DB.prepare('SELECT * FROM users').all(); +``` + +**✅ Supported adapters** (2026): +- **SvelteKit**: `@sveltejs/adapter-cloudflare` +- **Astro**: Built-in Cloudflare adapter +- **Nuxt**: Set `nitro.preset: 'cloudflare-pages'` in `nuxt.config.ts` +- **Qwik**: Built-in Cloudflare adapter +- **Solid Start**: `@solidjs/start-cloudflare-pages` + +**❌ Deprecated/Unsupported**: +- **Next.js**: Official adapter (`@cloudflare/next-on-pages`) deprecated. Use Vercel or self-host on Workers. +- **Remix**: Official adapter (`@remix-run/cloudflare-pages`) deprecated. Migrate to supported frameworks. + +See [gotchas.md](./gotchas.md#framework-specific) for migration guidance. diff --git a/plugins/cloudflare/skills/cloudflare/references/pages/configuration.md b/plugins/cloudflare/skills/cloudflare/references/pages/configuration.md new file mode 100644 index 0000000..30ada89 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages/configuration.md @@ -0,0 +1,201 @@ +# Configuration + +## wrangler.jsonc + +```jsonc +{ + "name": "my-pages-project", + "pages_build_output_dir": "./dist", + "compatibility_date": "2026-01-01", // Use current date for new projects + "compatibility_flags": ["nodejs_compat"], + "placement": { + "mode": "smart" // Optional: Enable Smart Placement + }, + "kv_namespaces": [{"binding": "KV", "id": "abcd1234..."}], + "d1_databases": [{"binding": "DB", "database_id": "xxxx-xxxx", "database_name": "production-db"}], + "r2_buckets": [{"binding": "BUCKET", "bucket_name": "my-bucket"}], + "durable_objects": {"bindings": [{"name": "COUNTER", "class_name": "Counter", "script_name": "counter-worker"}]}, + "services": [{"binding": "API", "service": "api-worker"}], + "queues": {"producers": [{"binding": "QUEUE", "queue": "my-queue"}]}, + "vectorize": [{"binding": "VECTORIZE", "index_name": "my-index"}], + "ai": {"binding": "AI"}, + "analytics_engine_datasets": [{"binding": "ANALYTICS"}], + "vars": {"API_URL": "https://api.example.com", "ENVIRONMENT": "production"}, + "env": { + "preview": { + "vars": {"API_URL": "https://staging-api.example.com"}, + "kv_namespaces": [{"binding": "KV", "id": "preview-namespace-id"}] + } + } +} +``` + +## Build Config + +**Git deployment**: Dashboard → Project → Settings → Build settings +Set build command, output dir, env vars. Framework auto-detection configures automatically. + +## Environment Variables + +### Local (.dev.vars) +```bash +# .dev.vars (never commit) +SECRET_KEY="local-secret-key" +API_TOKEN="dev-token-123" +``` + +### Production +```bash +echo "secret-value" | npx wrangler pages secret put SECRET_KEY --project-name=my-project +npx wrangler pages secret list --project-name=my-project +npx wrangler pages secret delete SECRET_KEY --project-name=my-project +``` + +Access: `env.SECRET_KEY` + +## Static Config Files + +### _redirects +Place in build output (e.g., `dist/_redirects`): + +```txt +/old-page /new-page 301 # 301 redirect +/blog/* /news/:splat 301 # Splat wildcard +/users/:id /members/:id 301 # Placeholders +/api/* /api-v2/:splat 200 # Proxy (no redirect) +``` + +**Limits**: 2,100 total (2,000 static + 100 dynamic), 1,000 char/line +**Note**: Functions take precedence + +### _headers +```txt +/secure/* + X-Frame-Options: DENY + X-Content-Type-Options: nosniff + +/api/* + Access-Control-Allow-Origin: * + +/static/* + Cache-Control: public, max-age=31536000, immutable +``` + +**Limits**: 100 rules, 2,000 char/line +**Note**: Only static assets; Functions set headers in Response + +### _routes.json +Controls which requests invoke Functions (auto-generated for most frameworks): + +```json +{ + "version": 1, + "include": ["/*"], + "exclude": ["/build/*", "/static/*", "/assets/*", "/*.{ico,png,jpg,css,js}"] +} +``` + +**Purpose**: Functions are metered; static requests are free. `exclude` takes precedence. Max 100 rules, 100 char/rule. + +## TypeScript + +```bash +npx wrangler types --path='./functions/types.d.ts' +``` + +Point `types` in `functions/tsconfig.json` to generated file. + +## Smart Placement + +Automatically optimizes function execution location based on request patterns. + +```jsonc +{ + "placement": { + "mode": "smart" // Enable optimization (default: off) + } +} +``` + +**How it works**: System analyzes traffic over hours/days and places function execution closer to: +- User clusters (e.g., regional traffic) +- Data sources (e.g., D1 database primary location) + +**Benefits**: +- Lower latency for read-heavy apps with centralized databases +- Better performance for apps with regional traffic patterns + +**Trade-offs**: +- Initial learning period: First requests may be slower while system optimizes +- Optimization time: Performance improves over 24-48 hours + +**When to enable**: Global apps with D1/Durable Objects in specific regions, or apps with concentrated geographic traffic. + +**When to skip**: Evenly distributed global traffic with no data locality constraints. + +## Remote Bindings (Local Dev) + +Connect local dev server to production bindings instead of local mocks: + +```bash +# All bindings remote +npx wrangler pages dev ./dist --remote + +# Specific bindings remote (others local) +npx wrangler pages dev ./dist --remote --kv=KV --d1=DB +``` + +**Use cases**: +- Test against production data (read-only operations) +- Debug binding-specific behavior +- Validate changes before deployment + +**⚠️ Warning**: +- Writes affect **real production data** +- Use only for read-heavy debugging or with non-production accounts +- Consider creating separate preview environments instead + +**Requirements**: Must be logged in (`npx wrangler login`) with access to bindings. + +## Local Dev + +```bash +# Basic +npx wrangler pages dev ./dist + +# With bindings +npx wrangler pages dev ./dist --kv KV --d1 DB=local-db-id + +# Remote bindings (production data) +npx wrangler pages dev ./dist --remote + +# Persistence +npx wrangler pages dev ./dist --persist-to=./.wrangler/state/v3 + +# Proxy mode (SSR frameworks) +npx wrangler pages dev -- npm run dev +``` + +## Limits (as of Jan 2026) + +| Resource | Free | Paid | +|----------|------|------| +| **Functions Requests** | 100k/day | Unlimited (metered) | +| **Function CPU Time** | 10ms/req | 30ms/req (Workers Paid) | +| **Function Memory** | 128MB | 128MB | +| **Script Size** | 1MB compressed | 10MB compressed | +| **Deployments** | 500/month | 5,000/month | +| **Files per Deploy** | 20,000 | 20,000 | +| **File Size** | 25MB | 25MB | +| **Build Time** | 20min | 20min | +| **Redirects** | 2,100 (2k static + 100 dynamic) | Same | +| **Header Rules** | 100 | 100 | +| **Route Rules** | 100 | 100 | +| **Subrequests** | 50/request | 1,000/request (Workers Paid) | + +**Notes**: +- Functions use Workers runtime; Workers Paid plan increases limits +- Free plan sufficient for most projects +- Static requests always free (not counted toward limits) + +[Full limits](https://developers.cloudflare.com/pages/platform/limits/) diff --git a/plugins/cloudflare/skills/cloudflare/references/pages/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/pages/gotchas.md new file mode 100644 index 0000000..943c2d3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages/gotchas.md @@ -0,0 +1,203 @@ +# Gotchas + +## Functions Not Running + +**Problem**: Function endpoints return 404 or don't execute +**Causes**: `_routes.json` excludes path; wrong file extension (`.jsx`/`.tsx`); Functions dir not at output root +**Solution**: Check `_routes.json`, rename to `.ts`/`.js`, verify build output structure + +## 404 on Static Assets + +**Problem**: Static files not serving +**Causes**: Build output dir misconfigured; Functions catching requests; Advanced mode missing `env.ASSETS.fetch()` +**Solution**: Verify output dir, add exclusions to `_routes.json`, call `env.ASSETS.fetch()` in `_worker.js` + +## Bindings Not Working + +**Problem**: `env.BINDING` undefined or errors +**Causes**: wrangler.jsonc syntax error; wrong binding IDs; missing `.dev.vars`; out-of-sync types +**Solution**: Validate config, verify IDs, create `.dev.vars`, run `npx wrangler types` + +## Build Failures + +**Problem**: Deployment fails during build +**Causes**: Wrong build command/output dir; Node version incompatibility; missing env vars; 20min timeout; OOM +**Solution**: Check Dashboard → Deployments → Build log; verify settings; add `.nvmrc`; optimize build + +## Middleware Not Running + +**Problem**: Middleware doesn't execute +**Causes**: Wrong filename (not `_middleware.ts`); missing `onRequest` export; didn't call `next()` +**Solution**: Rename file with underscore prefix; export handler; call `next()` or return Response + +## Headers/Redirects Not Working + +**Problem**: `_headers` or `_redirects` not applying +**Causes**: Only work for static assets; Functions override; syntax errors; exceeded limits +**Solution**: Set headers in Response object for Functions; verify syntax; check limits (100 headers, 2,100 redirects) + +## TypeScript Errors + +**Problem**: Type errors in Functions code +**Causes**: Types not generated; Env interface doesn't match wrangler.jsonc +**Solution**: Run `npx wrangler types --path='./functions/types.d.ts'`; update Env interface + +## Local Dev Issues + +**Problem**: Dev server errors or bindings don't work +**Causes**: Port conflict; bindings not passed; local vs HTTPS differences +**Solution**: Use `--port=3000`; pass bindings via CLI or wrangler.jsonc; account for HTTP/HTTPS differences + +## Performance Issues + +**Problem**: Slow responses or CPU limit errors +**Causes**: Functions invoked for static assets; cold starts; 10ms CPU limit; large bundle +**Solution**: Exclude static via `_routes.json`; optimize hot paths; keep bundle < 1MB + +## Framework-Specific + +### ⚠️ Deprecated Frameworks + +**Next.js**: Official adapter (`@cloudflare/next-on-pages`) **deprecated** and unmaintained. +- **Problem**: No updates since 2024; incompatible with Next.js 15+; missing App Router features +- **Cause**: Cloudflare discontinued official support; community fork exists but limited +- **Solutions**: + 1. **Recommended**: Use Vercel (official Next.js host) + 2. **Advanced**: Self-host on Workers using custom adapter (complex, unsupported) + 3. **Migration**: Switch to SvelteKit/Nuxt (similar DX, full Pages support) + +**Remix**: Official adapter (`@remix-run/cloudflare-pages`) **deprecated**. +- **Problem**: No maintenance from Remix team; compatibility issues with Remix v2+ +- **Cause**: Remix team deprecated all framework adapters +- **Solutions**: + 1. **Recommended**: Migrate to SvelteKit (similar file-based routing, better DX) + 2. **Alternative**: Use Astro (static-first with optional SSR) + 3. **Workaround**: Continue using deprecated adapter (no future support) + +### ✅ Supported Frameworks + +**SvelteKit**: +- Use `@sveltejs/adapter-cloudflare` +- Access bindings via `platform.env` in server load functions +- Set `platform: 'cloudflare'` in `svelte.config.js` + +**Astro**: +- Built-in Cloudflare adapter +- Access bindings via `Astro.locals.runtime.env` + +**Nuxt**: +- Set `nitro.preset: 'cloudflare-pages'` in `nuxt.config.ts` +- Access bindings via `event.context.cloudflare.env` + +**Qwik, Solid Start**: +- Built-in or official Cloudflare adapters available +- Check respective framework docs for binding access + +## Debugging + +```typescript +// Log request details +console.log('Request:', { method: request.method, url: request.url }); +console.log('Env:', Object.keys(env)); +console.log('Params:', params); +``` + +**View logs**: `npx wrangler pages deployment tail --project-name=my-project` + +## Smart Placement Issues + +### Increased Cold Start Latency + +**Problem**: First requests slower after enabling Smart Placement +**Cause**: Initial optimization period while system learns traffic patterns +**Solution**: Expected behavior during first 24-48 hours; monitor latency trends over time + +### Inconsistent Response Times + +**Problem**: Latency varies significantly across requests during initial deployment +**Cause**: Smart Placement testing different execution locations to find optimal placement +**Solution**: Normal during learning phase; stabilizes after traffic patterns emerge (1-2 days) + +### No Performance Improvement + +**Problem**: Smart Placement enabled but no latency reduction observed +**Cause**: Traffic evenly distributed globally, or no data locality constraints +**Solution**: Smart Placement most effective with centralized data (D1/DO) or regional traffic; disable if no benefit + +## Remote Bindings Issues + +### Accidentally Modified Production Data + +**Problem**: Local dev with `--remote` altered production database/KV +**Cause**: Remote bindings connect directly to production resources; writes are real +**Solution**: +- Use `--remote` only for read-heavy debugging +- Create separate preview environments for testing +- Never use `--remote` for write operations during development + +### Remote Binding Auth Errors + +**Problem**: `npx wrangler pages dev --remote` fails with "Unauthorized" or auth error +**Cause**: Not logged in, session expired, or insufficient account permissions +**Solution**: +1. Run `npx wrangler login` to re-authenticate +2. Verify account has access to project and bindings +3. Check binding IDs match production configuration + +### Slow Local Dev with Remote Bindings + +**Problem**: Local dev server slow when using `--remote` +**Cause**: Every request makes network calls to production bindings +**Solution**: Use local bindings for development; reserve `--remote` for final validation + +## Common Errors + +### "Module not found" +**Cause**: Dependencies not bundled or build output incorrect +**Solution**: Check build output directory, ensure dependencies bundled + +### "Binding not found" +**Cause**: Binding not configured or types out of sync +**Solution**: Verify wrangler.jsonc, run `npx wrangler types` + +### "Request exceeded CPU limit" +**Cause**: Code execution too slow or heavy compute +**Solution**: Optimize hot paths, upgrade to Workers Paid + +### "Script too large" +**Cause**: Bundle size exceeds limit +**Solution**: Tree-shake, use dynamic imports, code-split + +### "Too many subrequests" +**Cause**: Exceeded 50 subrequest limit +**Solution**: Batch or reduce fetch calls + +### "KV key not found" +**Cause**: Key doesn't exist or wrong namespace +**Solution**: Check namespace matches environment + +### "D1 error" +**Cause**: Wrong database_id or missing migrations +**Solution**: Verify config, run `wrangler d1 migrations list` + +## Limits Reference (Jan 2026) + +| Resource | Free | Paid | +|----------|------|------| +| Functions Requests | 100k/day | Unlimited | +| CPU Time | 10ms/req | 30ms/req | +| Memory | 128MB | 128MB | +| Script Size | 1MB | 10MB | +| Subrequests | 50/req | 1,000/req | +| Deployments | 500/month | 5,000/month | + +**Tip**: Hitting CPU limit? Optimize hot paths or upgrade to Workers Paid plan. + +[Full limits](https://developers.cloudflare.com/pages/platform/limits/) + +## Getting Help + +1. Check [Pages Docs](https://developers.cloudflare.com/pages/) +2. Search [Discord #functions](https://discord.com/channels/595317990191398933/910978223968518144) +3. Review [Workers Examples](https://developers.cloudflare.com/workers/examples/) +4. Check framework-specific docs/adapters diff --git a/plugins/cloudflare/skills/cloudflare/references/pages/patterns.md b/plugins/cloudflare/skills/cloudflare/references/pages/patterns.md new file mode 100644 index 0000000..883c4da --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pages/patterns.md @@ -0,0 +1,204 @@ +# Patterns + +## API Routes + +```typescript +// functions/api/todos/[id].ts +export const onRequestGet: PagesFunction = async ({ env, params }) => { + const todo = await env.DB.prepare('SELECT * FROM todos WHERE id = ?').bind(params.id).first(); + if (!todo) return new Response('Not found', { status: 404 }); + return Response.json(todo); +}; + +export const onRequestPut: PagesFunction = async ({ env, params, request }) => { + const body = await request.json(); + await env.DB.prepare('UPDATE todos SET title = ?, completed = ? WHERE id = ?') + .bind(body.title, body.completed, params.id).run(); + return Response.json({ success: true }); +}; +// Also: onRequestDelete, onRequestPost +``` + +## Auth Middleware + +```typescript +// functions/_middleware.ts +const auth: PagesFunction = async (context) => { + if (context.request.url.includes('/public/')) return context.next(); + const authHeader = context.request.headers.get('Authorization'); + if (!authHeader?.startsWith('Bearer ')) { + return new Response('Unauthorized', { status: 401 }); + } + + try { + const payload = await verifyJWT(authHeader.substring(7), context.env.JWT_SECRET); + context.data.user = payload; + return context.next(); + } catch (err) { + return new Response('Invalid token', { status: 401 }); + } +}; +export const onRequest = [auth]; +``` + +## CORS + +```typescript +// functions/api/_middleware.ts +const corsHeaders = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization' +}; + +export const onRequest: PagesFunction = async (context) => { + if (context.request.method === 'OPTIONS') { + return new Response(null, {headers: corsHeaders}); + } + const response = await context.next(); + Object.entries(corsHeaders).forEach(([k, v]) => response.headers.set(k, v)); + return response; +}; +``` + +## Form Handling + +```typescript +// functions/api/contact.ts +export const onRequestPost: PagesFunction = async ({ request, env }) => { + const formData = await request.formData(); + await env.QUEUE.send({name: formData.get('name'), email: formData.get('email')}); + return new Response('

Thanks!

', { headers: { 'Content-Type': 'text/html' } }); +}; +``` + +## Background Tasks + +```typescript +export const onRequestPost: PagesFunction = async ({ request, waitUntil }) => { + const data = await request.json(); + waitUntil(fetch('https://api.example.com/webhook', { + method: 'POST', body: JSON.stringify(data) + })); + return Response.json({ queued: true }); +}; +``` + +## Error Handling + +```typescript +// functions/_middleware.ts +const errorHandler: PagesFunction = async (context) => { + try { + return await context.next(); + } catch (error) { + console.error('Error:', error); + if (context.request.url.includes('/api/')) { + return Response.json({ error: error.message }, { status: 500 }); + } + return new Response(`

Error

${error.message}

`, { + status: 500, headers: { 'Content-Type': 'text/html' } + }); + } +}; +export const onRequest = [errorHandler]; +``` + +## Caching + +```typescript +// functions/api/data.ts +export const onRequestGet: PagesFunction = async ({ env, request }) => { + const cacheKey = `data:${new URL(request.url).pathname}`; + const cached = await env.KV.get(cacheKey, 'json'); + if (cached) return Response.json(cached, { headers: { 'X-Cache': 'HIT' } }); + + const data = await env.DB.prepare('SELECT * FROM data').first(); + await env.KV.put(cacheKey, JSON.stringify(data), {expirationTtl: 3600}); + return Response.json(data, {headers: {'X-Cache': 'MISS'}}); +}; +``` + +## Smart Placement for Database Apps + +Enable Smart Placement for apps with D1 or centralized data sources: + +```jsonc +// wrangler.jsonc +{ + "name": "global-app", + "placement": { + "mode": "smart" + }, + "d1_databases": [{ + "binding": "DB", + "database_id": "your-db-id" + }] +} +``` + +```typescript +// functions/api/data.ts +export const onRequestGet: PagesFunction = async ({ env }) => { + // Smart Placement optimizes execution location over time + // Balances user location vs database location + const data = await env.DB.prepare('SELECT * FROM products LIMIT 10').all(); + return Response.json(data); +}; +``` + +**Best for**: Read-heavy apps with D1/Durable Objects in specific regions. +**Not needed**: Apps without data locality constraints or with evenly distributed traffic. + +## Framework Integration + +**Supported** (2026): SvelteKit, Astro, Nuxt, Qwik, Solid Start + +```bash +npm create cloudflare@latest my-app -- --framework=svelte +``` + +### SvelteKit +```typescript +// src/routes/+page.server.ts +export const load = async ({ platform }) => { + const todos = await platform.env.DB.prepare('SELECT * FROM todos').all(); + return { todos: todos.results }; +}; +``` + +### Astro +```astro +--- +const { DB } = Astro.locals.runtime.env; +const todos = await DB.prepare('SELECT * FROM todos').all(); +--- +
    {todos.results.map(t =>
  • {t.title}
    • )}
+``` + +### Nuxt +```typescript +// server/api/todos.get.ts +export default defineEventHandler(async (event) => { + const { DB } = event.context.cloudflare.env; + return await DB.prepare('SELECT * FROM todos').all(); +}); +``` + +**⚠️ Framework Status** (2026): +- ✅ **Supported**: SvelteKit, Astro, Nuxt, Qwik, Solid Start +- ❌ **Deprecated**: Next.js (`@cloudflare/next-on-pages`), Remix (`@remix-run/cloudflare-pages`) + +For deprecated frameworks, see [gotchas.md](./gotchas.md#framework-specific) for migration options. + +[Framework Guides](https://developers.cloudflare.com/pages/framework-guides/) + +## Monorepo + +Dashboard → Settings → Build → Root directory. Set to subproject (e.g., `apps/web`). + +## Best Practices + +**Performance**: Exclude static via `_routes.json`; cache with KV; keep bundle < 1MB +**Security**: Use secrets (not vars); validate inputs; rate limit with KV/DO +**Workflow**: Preview per branch; local dev with `wrangler pages dev`; instant rollbacks in Dashboard diff --git a/plugins/cloudflare/skills/cloudflare/references/pipelines/README.md b/plugins/cloudflare/skills/cloudflare/references/pipelines/README.md new file mode 100644 index 0000000..2724485 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pipelines/README.md @@ -0,0 +1,105 @@ +# Cloudflare Pipelines + +ETL streaming platform for ingesting, transforming, and loading data into R2 with SQL transformations. + +## Overview + +Pipelines provides: +- **Streams**: Durable event buffers (HTTP/Workers ingestion) +- **Pipelines**: SQL-based transformations +- **Sinks**: R2 destinations (Iceberg tables or Parquet/JSON files) + +**Status**: Open beta (Workers Paid plan) +**Pricing**: No charge beyond standard R2 storage/operations + +## Architecture + +``` +Data Sources → Streams → Pipelines (SQL) → Sinks → R2 + ↑ ↓ ↓ + HTTP/Workers Transform Iceberg/Parquet +``` + +| Component | Purpose | Key Feature | +|-----------|---------|-------------| +| Streams | Event ingestion | Structured (validated) or unstructured | +| Pipelines | Transform with SQL | Immutable after creation | +| Sinks | Write to R2 | Exactly-once delivery | + +## Quick Start + +```bash +# Interactive setup (recommended) +npx wrangler pipelines setup +``` + +**Minimal Worker example:** +```typescript +interface Env { + STREAM: Pipeline; +} + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const event = { user_id: "123", event_type: "purchase", amount: 29.99 }; + + // Fire-and-forget pattern + ctx.waitUntil(env.STREAM.send([event])); + + return new Response('OK'); + } +} satisfies ExportedHandler; +``` + +## Which Sink Type? + +``` +Need SQL queries on data? + → R2 Data Catalog (Iceberg) + ✅ ACID transactions, time-travel, schema evolution + ❌ More setup complexity (namespace, table, catalog token) + +Just file storage/archival? + → R2 Storage (Parquet) + ✅ Simple, direct file access + ❌ No built-in SQL queries + +Using external tools (Spark/Athena)? + → R2 Storage (Parquet with partitioning) + ✅ Standard format, partition pruning for performance + ❌ Must manage schema compatibility yourself +``` + +## Common Use Cases + +- **Analytics pipelines**: Clickstream, telemetry, server logs +- **Data warehousing**: ETL into queryable Iceberg tables +- **Event processing**: Mobile/IoT with enrichment +- **Ecommerce analytics**: User events, purchases, views + +## Reading Order + +**New to Pipelines?** Start here: +1. [configuration.md](./configuration.md) - Setup streams, sinks, pipelines +2. [api.md](./api.md) - Send events, TypeScript types, SQL functions +3. [patterns.md](./patterns.md) - Best practices, integrations, complete example +4. [gotchas.md](./gotchas.md) - Critical warnings, troubleshooting + +**Task-based routing:** +- Setup pipeline → [configuration.md](./configuration.md) +- Send/query data → [api.md](./api.md) +- Implement pattern → [patterns.md](./patterns.md) +- Debug issue → [gotchas.md](./gotchas.md) + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc bindings, schema definition, sink options, CLI commands +- [api.md](./api.md) - Pipeline binding interface, send() method, HTTP ingest, SQL function reference +- [patterns.md](./patterns.md) - Fire-and-forget, schema validation with Zod, integrations, performance tuning +- [gotchas.md](./gotchas.md) - Silent validation failures, immutable pipelines, latency expectations, limits + +## See Also + +- [r2](../r2/) - R2 storage backend for sinks +- [queues](../queues/) - Compare with Queues for async processing +- [workers](../workers/) - Worker runtime for event ingestion diff --git a/plugins/cloudflare/skills/cloudflare/references/pipelines/api.md b/plugins/cloudflare/skills/cloudflare/references/pipelines/api.md new file mode 100644 index 0000000..ff302c7 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pipelines/api.md @@ -0,0 +1,208 @@ +# Pipelines API Reference + +## Pipeline Binding Interface + +```typescript +// From @cloudflare/workers-types +interface Pipeline { + send(data: object | object[]): Promise; +} + +interface Env { + STREAM: Pipeline; +} + +export default { + async fetch(request: Request, env: Env): Promise { + // send() returns Promise - no result data + await env.STREAM.send([event]); + return new Response('OK'); + } +} satisfies ExportedHandler; +``` + +**Key points:** +- `send()` accepts single object or array +- Always returns `Promise` (no confirmation data) +- Throws on network/validation errors (wrap in try/catch) +- Use `ctx.waitUntil()` for fire-and-forget pattern + +## Writing Events + +### Single Event + +```typescript +await env.STREAM.send([{ + user_id: "12345", + event_type: "purchase", + product_id: "widget-001", + amount: 29.99 +}]); +``` + +### Batch Events + +```typescript +const events = [ + { user_id: "user1", event_type: "view" }, + { user_id: "user2", event_type: "purchase", amount: 50 } +]; +await env.STREAM.send(events); +``` + +**Limits:** +- Max 1 MB per request +- 5 MB/s per stream + +### Fire-and-Forget Pattern + +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const event = { /* ... */ }; + + // Don't block response on send + ctx.waitUntil(env.STREAM.send([event])); + + return new Response('OK'); + } +}; +``` + +### Error Handling + +```typescript +try { + await env.STREAM.send([event]); +} catch (error) { + console.error('Pipeline send failed:', error); + // Log to another system, retry, or return error response + return new Response('Failed to track event', { status: 500 }); +} +``` + +## HTTP Ingest API + +### Endpoint Format + +``` +https://{stream-id}.ingest.cloudflare.com +``` + +Get `{stream-id}` from: `npx wrangler pipelines streams list` + +### Request Format + +**CRITICAL:** Must send array, not single object + +```bash +# ✅ Correct +curl -X POST https://{stream-id}.ingest.cloudflare.com \ + -H "Content-Type: application/json" \ + -d '[{"user_id": "123", "event_type": "purchase"}]' + +# ❌ Wrong - will fail +curl -X POST https://{stream-id}.ingest.cloudflare.com \ + -H "Content-Type: application/json" \ + -d '{"user_id": "123", "event_type": "purchase"}' +``` + +### Authentication + +```bash +curl -X POST https://{stream-id}.ingest.cloudflare.com \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_TOKEN" \ + -d '[{"event": "data"}]' +``` + +**Required permission:** Workers Pipeline Send + +Create token: Dashboard → Workers → API tokens → Create with Pipeline Send permission + +### Response Codes + +| Code | Meaning | Action | +|------|---------|--------| +| 200 | Accepted | Success | +| 400 | Invalid format | Check JSON array, schema match | +| 401 | Auth failed | Verify token valid | +| 413 | Payload too large | Split into smaller batches (<1 MB) | +| 429 | Rate limited | Back off, retry with delay | +| 5xx | Server error | Retry with exponential backoff | + +## SQL Functions Quick Reference + +Available in `INSERT INTO sink SELECT ... FROM stream` transformations: + +| Function | Example | Use Case | +|----------|---------|----------| +| `UPPER(s)` | `UPPER(event_type)` | Normalize strings | +| `LOWER(s)` | `LOWER(email)` | Case-insensitive matching | +| `CONCAT(...)` | `CONCAT(user_id, '_', product_id)` | Generate composite keys | +| `CASE WHEN ... THEN ... END` | `CASE WHEN amount > 100 THEN 'high' ELSE 'low' END` | Conditional enrichment | +| `CAST(x AS type)` | `CAST(timestamp AS string)` | Type conversion | +| `COALESCE(x, y)` | `COALESCE(amount, 0.0)` | Default values | +| Math operators | `amount * 1.1`, `price / quantity` | Calculations | +| Comparison | `amount > 100`, `status IN ('active', 'pending')` | Filtering | + +**String types for CAST:** `string`, `int32`, `int64`, `float32`, `float64`, `bool`, `timestamp` + +Full reference: [Pipelines SQL Reference](https://developers.cloudflare.com/pipelines/sql-reference/) + +## SQL Transform Examples + +### Filter Events + +```sql +INSERT INTO my_sink +SELECT * FROM my_stream +WHERE event_type = 'purchase' AND amount > 100 +``` + +### Select Specific Fields + +```sql +INSERT INTO my_sink +SELECT user_id, event_type, timestamp, amount +FROM my_stream +``` + +### Transform and Enrich + +```sql +INSERT INTO my_sink +SELECT + user_id, + UPPER(event_type) as event_type, + timestamp, + amount * 1.1 as amount_with_tax, + CONCAT(user_id, '_', product_id) as unique_key, + CASE + WHEN amount > 1000 THEN 'high_value' + WHEN amount > 100 THEN 'medium_value' + ELSE 'low_value' + END as customer_tier +FROM my_stream +WHERE event_type IN ('purchase', 'refund') +``` + +## Querying Results (R2 Data Catalog) + +```bash +export WRANGLER_R2_SQL_AUTH_TOKEN=YOUR_CATALOG_TOKEN + +npx wrangler r2 sql query "warehouse_name" " +SELECT + event_type, + COUNT(*) as event_count, + SUM(amount) as total_revenue +FROM default.my_table +WHERE event_type = 'purchase' + AND timestamp >= '2025-01-01' +GROUP BY event_type +ORDER BY total_revenue DESC +LIMIT 100" +``` + +**Note:** Iceberg tables support standard SQL queries with GROUP BY, JOINs, WHERE, ORDER BY, etc. diff --git a/plugins/cloudflare/skills/cloudflare/references/pipelines/configuration.md b/plugins/cloudflare/skills/cloudflare/references/pipelines/configuration.md new file mode 100644 index 0000000..75e65f5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pipelines/configuration.md @@ -0,0 +1,98 @@ +# Pipelines Configuration + +## Worker Binding + +```jsonc +// wrangler.jsonc +{ + "pipelines": [ + { "pipeline": "", "binding": "STREAM" } + ] +} +``` + +Get stream ID: `npx wrangler pipelines streams list` + +## Schema (Structured Streams) + +```json +{ + "fields": [ + { "name": "user_id", "type": "string", "required": true }, + { "name": "event_type", "type": "string", "required": true }, + { "name": "amount", "type": "float64", "required": false }, + { "name": "timestamp", "type": "timestamp", "required": true } + ] +} +``` + +**Types:** `string`, `int32`, `int64`, `float32`, `float64`, `bool`, `timestamp`, `json`, `binary`, `list`, `struct` + +## Stream Setup + +```bash +# With schema +npx wrangler pipelines streams create my-stream --schema-file schema.json + +# Unstructured (no validation) +npx wrangler pipelines streams create my-stream + +# List/get/delete +npx wrangler pipelines streams list +npx wrangler pipelines streams get +npx wrangler pipelines streams delete +``` + +## Sink Configuration + +**R2 Data Catalog (Iceberg):** +```bash +npx wrangler pipelines sinks create my-sink \ + --type r2-data-catalog \ + --bucket my-bucket --namespace default --table events \ + --catalog-token $TOKEN \ + --compression zstd --roll-interval 60 +``` + +**R2 Raw (Parquet):** +```bash +npx wrangler pipelines sinks create my-sink \ + --type r2 --bucket my-bucket --format parquet \ + --path analytics/events \ + --partitioning "year=%Y/month=%m/day=%d" \ + --access-key-id $KEY --secret-access-key $SECRET +``` + +| Option | Values | Guidance | +|--------|--------|----------| +| `--compression` | `zstd`, `snappy`, `gzip` | `zstd` best ratio, `snappy` fastest | +| `--roll-interval` | Seconds | Low latency: 10-60, Query perf: 300 | +| `--roll-size` | MB | Larger = better compression | + +## Pipeline Creation + +```bash +npx wrangler pipelines create my-pipeline \ + --sql "INSERT INTO my_sink SELECT * FROM my_stream WHERE event_type = 'purchase'" +``` + +**⚠️ Pipelines are immutable** - cannot modify SQL. Must delete/recreate. + +## Credentials + +| Type | Permission | Get From | +|------|------------|----------| +| Catalog token | R2 Admin Read & Write | Dashboard → R2 → API tokens | +| R2 credentials | Object Read & Write | `wrangler r2 bucket create` output | +| HTTP ingest token | Workers Pipeline Send | Dashboard → Workers → API tokens | + +## Complete Example + +```bash +npx wrangler r2 bucket create my-bucket +npx wrangler r2 bucket catalog enable my-bucket +npx wrangler pipelines streams create my-stream --schema-file schema.json +npx wrangler pipelines sinks create my-sink --type r2-data-catalog --bucket my-bucket ... +npx wrangler pipelines create my-pipeline --sql "INSERT INTO my_sink SELECT * FROM my_stream" +npx wrangler deploy +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/pipelines/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/pipelines/gotchas.md new file mode 100644 index 0000000..2a2a75f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pipelines/gotchas.md @@ -0,0 +1,80 @@ +# Pipelines Gotchas + +## Critical Issues + +### Events Silently Dropped + +**Most common issue.** Events accepted (HTTP 200) but never appear in sink. + +**Causes:** +1. Schema validation fails - structured streams drop invalid events silently +2. Waiting for roll interval (10-300s) - expected behavior + +**Solution:** Validate client-side with Zod: +```typescript +const EventSchema = z.object({ user_id: z.string(), amount: z.number() }); +try { + const validated = EventSchema.parse(rawEvent); + await env.STREAM.send([validated]); +} catch (e) { /* get immediate feedback */ } +``` + +### Pipelines Are Immutable + +Cannot modify SQL after creation. Must delete and recreate. + +```bash +npx wrangler pipelines delete old-pipeline +npx wrangler pipelines create new-pipeline --sql "..." +``` + +**Tip:** Use version naming (`events-pipeline-v1`) and keep SQL in version control. + +### Worker Binding Not Found + +**`env.STREAM is undefined`** + +1. Use **stream ID** (not pipeline ID) in `wrangler.jsonc` +2. Redeploy after adding binding + +```bash +npx wrangler pipelines streams list # Get stream ID +npx wrangler deploy +``` + +## Common Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| Events not in R2 | Roll interval not elapsed | Wait 10-300s, check `roll_interval` | +| Schema validation failures | Type mismatch, missing fields | Validate client-side | +| Rate limit (429) | >5 MB/s per stream | Batch events, request increase | +| Payload too large (413) | >1 MB request | Split into smaller batches | +| Cannot delete stream | Pipeline references it | Delete pipelines first | +| Sink credential errors | Token expired | Recreate sink with new credentials | + +## Limits (Open Beta) + +| Resource | Limit | +|----------|-------| +| Streams/Sinks/Pipelines per account | 20 each | +| Payload size | 1 MB | +| Ingest rate per stream | 5 MB/s | +| Event retention | 24 hours | +| Recommended batch size | 100 events | + +## SQL Limitations + +- **No JOINs** - single stream per pipeline +- **No window functions** - basic SQL only +- **No subqueries** - must use `INSERT INTO ... SELECT ... FROM` +- **No schema evolution** - cannot modify after creation + +## Debug Checklist + +- [ ] Stream exists: `npx wrangler pipelines streams list` +- [ ] Pipeline healthy: `npx wrangler pipelines get ` +- [ ] SQL syntax matches schema +- [ ] Worker redeployed after binding added +- [ ] Waited for roll interval +- [ ] Accepted vs processed count matches (no validation drops) diff --git a/plugins/cloudflare/skills/cloudflare/references/pipelines/patterns.md b/plugins/cloudflare/skills/cloudflare/references/pipelines/patterns.md new file mode 100644 index 0000000..186b6a2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pipelines/patterns.md @@ -0,0 +1,87 @@ +# Pipelines Patterns + +## Fire-and-Forget + +```typescript +export default { + async fetch(request, env, ctx) { + const event = { user_id: '...', event_type: 'page_view', timestamp: new Date().toISOString() }; + ctx.waitUntil(env.STREAM.send([event])); // Don't block response + return new Response('OK'); + } +}; +``` + +## Schema Validation with Zod + +```typescript +import { z } from 'zod'; + +const EventSchema = z.object({ + user_id: z.string(), + event_type: z.enum(['purchase', 'view']), + amount: z.number().positive().optional() +}); + +const validated = EventSchema.parse(rawEvent); // Throws on invalid +await env.STREAM.send([validated]); +``` + +**Why:** Structured streams drop invalid events silently. Client validation gives immediate feedback. + +## SQL Transform Patterns + +```sql +-- Filter early (reduce storage) +INSERT INTO my_sink +SELECT user_id, event_type, amount +FROM my_stream +WHERE event_type = 'purchase' AND amount > 10 + +-- Select only needed fields +INSERT INTO my_sink +SELECT user_id, event_type, timestamp FROM my_stream + +-- Enrich with CASE +INSERT INTO my_sink +SELECT user_id, amount, + CASE WHEN amount > 1000 THEN 'vip' ELSE 'standard' END as tier +FROM my_stream +``` + +## Pipelines + Queues Fan-out + +```typescript +await Promise.all([ + env.ANALYTICS_STREAM.send([event]), // Long-term storage + env.PROCESS_QUEUE.send(event) // Immediate processing +]); +``` + +| Need | Use | +|------|-----| +| Long-term storage, SQL queries | Pipelines | +| Immediate processing, retries | Queues | +| Both | Fan-out pattern | + +## Performance Tuning + +| Goal | Config | +|------|--------| +| Low latency | `--roll-interval 10` | +| Query performance | `--roll-interval 300 --roll-size 100` | +| Cost optimal | `--compression zstd --roll-interval 300` | + +## Schema Evolution + +Pipelines are immutable. Use versioning: + +```bash +# Create v2 stream/sink/pipeline +npx wrangler pipelines streams create events-v2 --schema-file v2.json + +# Dual-write during transition +await Promise.all([env.EVENTS_V1.send([event]), env.EVENTS_V2.send([event])]); + +# Query across versions with UNION ALL +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/pulumi/README.md b/plugins/cloudflare/skills/cloudflare/references/pulumi/README.md new file mode 100644 index 0000000..e78d807 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pulumi/README.md @@ -0,0 +1,100 @@ +# Cloudflare Pulumi Provider + +Expert guidance for Cloudflare Pulumi Provider (@pulumi/cloudflare). + +## Overview + +Programmatic management of Cloudflare resources: Workers, Pages, D1, KV, R2, DNS, Queues, etc. + +**Packages:** +- TypeScript/JS: `@pulumi/cloudflare` +- Python: `pulumi-cloudflare` +- Go: `github.com/pulumi/pulumi-cloudflare/sdk/v6/go/cloudflare` +- .NET: `Pulumi.Cloudflare` + +**Version:** v6.x + +## Core Principles + +1. Use API tokens (not legacy API keys) +2. Store accountId in stack config +3. Match binding names across code/config +4. Use `module: true` for ES modules +5. Set `compatibilityDate` to lock behavior + +## Authentication + +```typescript +import * as cloudflare from "@pulumi/cloudflare"; + +// API Token (recommended): CLOUDFLARE_API_TOKEN env +const provider = new cloudflare.Provider("cf", { apiToken: process.env.CLOUDFLARE_API_TOKEN }); + +// API Key (legacy): CLOUDFLARE_API_KEY + CLOUDFLARE_EMAIL env +const provider = new cloudflare.Provider("cf", { apiKey: process.env.CLOUDFLARE_API_KEY, email: process.env.CLOUDFLARE_EMAIL }); + +// API User Service Key: CLOUDFLARE_API_USER_SERVICE_KEY env +const provider = new cloudflare.Provider("cf", { apiUserServiceKey: process.env.CLOUDFLARE_API_USER_SERVICE_KEY }); +``` + +## Setup + +**Pulumi.yaml:** +```yaml +name: my-cloudflare-app +runtime: nodejs +config: + cloudflare:apiToken: + value: ${CLOUDFLARE_API_TOKEN} +``` + +**Pulumi..yaml:** +```yaml +config: + cloudflare:accountId: "abc123..." +``` + +**index.ts:** +```typescript +import * as pulumi from "@pulumi/pulumi"; +import * as cloudflare from "@pulumi/cloudflare"; +const accountId = new pulumi.Config("cloudflare").require("accountId"); +``` + +## Common Resource Types +- `Provider` - Provider config +- `WorkerScript` - Worker +- `WorkersKvNamespace` - KV +- `R2Bucket` - R2 +- `D1Database` - D1 +- `Queue` - Queue +- `PagesProject` - Pages +- `DnsRecord` - DNS +- `WorkerRoute` - Worker route +- `WorkersDomain` - Custom domain + +## Key Properties +- `accountId` - Required for most resources +- `zoneId` - Required for DNS/domain +- `name`/`title` - Resource identifier +- `*Bindings` - Connect resources to Workers + +## Reading Order + +| Order | File | What | When to Read | +|-------|------|------|--------------| +| 1 | [configuration.md](./configuration.md) | Resource config for Workers/KV/D1/R2/Queues/Pages | First time setup, resource reference | +| 2 | [patterns.md](./patterns.md) | Architecture patterns, multi-env, component resources | Building complex apps, best practices | +| 3 | [api.md](./api.md) | Outputs, dependencies, imports, dynamic providers | Advanced features, integrations | +| 4 | [gotchas.md](./gotchas.md) | Common errors, troubleshooting, limits | Debugging, deployment issues | + +## In This Reference +- [configuration.md](./configuration.md) - Provider config, stack setup, Workers/bindings +- [api.md](./api.md) - Resource types, Workers script, KV/D1/R2/queues/Pages +- [patterns.md](./patterns.md) - Multi-env, secrets, CI/CD, stack management +- [gotchas.md](./gotchas.md) - State issues, deployment failures, limits + +## See Also +- [terraform](../terraform/) - Alternative IaC for Cloudflare +- [wrangler](../wrangler/) - CLI deployment alternative +- [workers](../workers/) - Worker runtime documentation diff --git a/plugins/cloudflare/skills/cloudflare/references/pulumi/api.md b/plugins/cloudflare/skills/cloudflare/references/pulumi/api.md new file mode 100644 index 0000000..332cfef --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pulumi/api.md @@ -0,0 +1,200 @@ +# API & Data Sources + +## Outputs and Exports + +Export resource identifiers: + +```typescript +export const kvId = kv.id; +export const bucketName = bucket.name; +export const workerUrl = worker.subdomain; +export const dbId = db.id; +``` + +## Resource Dependencies + +Implicit dependencies via outputs: + +```typescript +const kv = new cloudflare.WorkersKvNamespace("kv", { + accountId: accountId, + title: "my-kv", +}); + +// Worker depends on KV (implicit via kv.id) +const worker = new cloudflare.WorkerScript("worker", { + accountId: accountId, + name: "my-worker", + content: code, + kvNamespaceBindings: [{name: "MY_KV", namespaceId: kv.id}], // Creates dependency +}); +``` + +Explicit dependencies: + +```typescript +const migration = new command.local.Command("migration", { + create: pulumi.interpolate`wrangler d1 execute ${db.name} --file ./schema.sql`, +}, {dependsOn: [db]}); + +const worker = new cloudflare.WorkerScript("worker", { + accountId: accountId, + name: "worker", + content: code, + d1DatabaseBindings: [{name: "DB", databaseId: db.id}], +}, {dependsOn: [migration]}); // Ensure migrations run first +``` + +## Using Outputs with API Calls + +```typescript +const db = new cloudflare.D1Database("db", {accountId, name: "my-db"}); + +db.id.apply(async (dbId) => { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/d1/database/${dbId}/query`, + {method: "POST", headers: {"Authorization": `Bearer ${apiToken}`, "Content-Type": "application/json"}, + body: JSON.stringify({sql: "CREATE TABLE users (id INT)"})} + ); + return response.json(); +}); +``` + +## Custom Dynamic Providers + +For resources not in provider: + +```typescript +import * as pulumi from "@pulumi/pulumi"; + +class D1MigrationProvider implements pulumi.dynamic.ResourceProvider { + async create(inputs: any): Promise { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${inputs.accountId}/d1/database/${inputs.databaseId}/query`, + {method: "POST", headers: {"Authorization": `Bearer ${inputs.apiToken}`, "Content-Type": "application/json"}, + body: JSON.stringify({sql: inputs.sql})} + ); + return {id: `${inputs.databaseId}-${Date.now()}`, outs: await response.json()}; + } + async update(id: string, olds: any, news: any): Promise { + if (olds.sql !== news.sql) await this.create(news); + return {}; + } + async delete(id: string, props: any): Promise {} +} + +class D1Migration extends pulumi.dynamic.Resource { + constructor(name: string, args: any, opts?: pulumi.CustomResourceOptions) { + super(new D1MigrationProvider(), name, args, opts); + } +} + +const migration = new D1Migration("migration", { + accountId, databaseId: db.id, apiToken, sql: "CREATE TABLE users (id INT)", +}, {dependsOn: [db]}); +``` + +## Data Sources + +**Get Zone:** +```typescript +const zone = cloudflare.getZone({name: "example.com"}); +const zoneId = zone.then(z => z.id); +``` + +**Get Accounts (via API):** +Use Cloudflare API directly or custom dynamic resources. + +## Import Existing Resources + +```bash +# Import worker +pulumi import cloudflare:index/workerScript:WorkerScript my-worker / + +# Import KV namespace +pulumi import cloudflare:index/workersKvNamespace:WorkersKvNamespace my-kv + +# Import R2 bucket +pulumi import cloudflare:index/r2Bucket:R2Bucket my-bucket / + +# Import D1 database +pulumi import cloudflare:index/d1Database:D1Database my-db / + +# Import DNS record +pulumi import cloudflare:index/dnsRecord:DnsRecord my-record / +``` + +## Secrets Management + +```typescript +import * as pulumi from "@pulumi/pulumi"; + +const config = new pulumi.Config(); +const apiKey = config.requireSecret("apiKey"); // Encrypted in state + +const worker = new cloudflare.WorkerScript("worker", { + accountId: accountId, + name: "my-worker", + content: code, + secretTextBindings: [{name: "API_KEY", text: apiKey}], +}); +``` + +Store secrets: +```bash +pulumi config set --secret apiKey "secret-value" +``` + +## Transform Pattern + +Modify resource args before creation: + +```typescript +import {Transform} from "@pulumi/pulumi"; + +interface BucketArgs { + accountId: pulumi.Input; + transform?: {bucket?: Transform}; +} + +function createBucket(name: string, args: BucketArgs) { + const bucketArgs: cloudflare.R2BucketArgs = { + accountId: args.accountId, + name: name, + location: "auto", + }; + const finalArgs = args.transform?.bucket?.(bucketArgs) ?? bucketArgs; + return new cloudflare.R2Bucket(name, finalArgs); +} +``` + +## v6.x Worker Versioning Resources + +**Worker** - Container for versions: +```typescript +const worker = new cloudflare.Worker("api", {accountId, name: "api-worker"}); +export const workerId = worker.id; +``` + +**WorkerVersion** - Immutable code + config: +```typescript +const version = new cloudflare.WorkerVersion("v1", { + accountId, workerId: worker.id, + content: fs.readFileSync("./dist/worker.js", "utf8"), + compatibilityDate: "2025-01-01", +}); +export const versionId = version.id; +``` + +**WorkersDeployment** - Active deployment with bindings: +```typescript +const deployment = new cloudflare.WorkersDeployment("prod", { + accountId, workerId: worker.id, versionId: version.id, + kvNamespaceBindings: [{name: "MY_KV", namespaceId: kv.id}], +}); +``` + +**Use:** Advanced deployments (canary, blue-green). Most apps should use `WorkerScript` (auto-versioning). + +--- +See: [README.md](./README.md), [configuration.md](./configuration.md), [patterns.md](./patterns.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/pulumi/configuration.md b/plugins/cloudflare/skills/cloudflare/references/pulumi/configuration.md new file mode 100644 index 0000000..449419d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pulumi/configuration.md @@ -0,0 +1,198 @@ +# Resource Configuration + +## Workers (cloudflare.WorkerScript) + +```typescript +import * as cloudflare from "@pulumi/cloudflare"; +import * as fs from "fs"; + +const worker = new cloudflare.WorkerScript("my-worker", { + accountId: accountId, + name: "my-worker", + content: fs.readFileSync("./dist/worker.js", "utf8"), + module: true, // ES modules + compatibilityDate: "2025-01-01", + compatibilityFlags: ["nodejs_compat"], + + // v6.x: Observability + logpush: true, // Enable Workers Logpush + tailConsumers: [{service: "log-consumer"}], // Stream logs to Worker + + // v6.x: Placement + placement: {mode: "smart"}, // Smart placement for latency optimization + + // Bindings + kvNamespaceBindings: [{name: "MY_KV", namespaceId: kv.id}], + r2BucketBindings: [{name: "MY_BUCKET", bucketName: bucket.name}], + d1DatabaseBindings: [{name: "DB", databaseId: db.id}], + queueBindings: [{name: "MY_QUEUE", queue: queue.id}], + serviceBindings: [{name: "OTHER_SERVICE", service: other.name}], + plainTextBindings: [{name: "ENV_VAR", text: "value"}], + secretTextBindings: [{name: "API_KEY", text: secret}], + + // v6.x: Advanced bindings + analyticsEngineBindings: [{name: "ANALYTICS", dataset: "my-dataset"}], + browserBinding: {name: "BROWSER"}, // Browser Rendering + aiBinding: {name: "AI"}, // Workers AI + hyperdriveBindings: [{name: "HYPERDRIVE", id: hyperdriveConfig.id}], +}); +``` + +## Workers KV (cloudflare.WorkersKvNamespace) + +```typescript +const kv = new cloudflare.WorkersKvNamespace("my-kv", { + accountId: accountId, + title: "my-kv-namespace", +}); + +// Write values +const kvValue = new cloudflare.WorkersKvValue("config", { + accountId: accountId, + namespaceId: kv.id, + key: "config", + value: JSON.stringify({foo: "bar"}), +}); +``` + +## R2 Buckets (cloudflare.R2Bucket) + +```typescript +const bucket = new cloudflare.R2Bucket("my-bucket", { + accountId: accountId, + name: "my-bucket", + location: "auto", // or "wnam", etc. +}); +``` + +## D1 Databases (cloudflare.D1Database) + +```typescript +const db = new cloudflare.D1Database("my-db", {accountId, name: "my-database"}); + +// Migrations via wrangler +import * as command from "@pulumi/command"; +const migration = new command.local.Command("d1-migration", { + create: pulumi.interpolate`wrangler d1 execute ${db.name} --file ./schema.sql`, +}, {dependsOn: [db]}); +``` + +## Queues (cloudflare.Queue) + +```typescript +const queue = new cloudflare.Queue("my-queue", {accountId, name: "my-queue"}); + +// Producer +const producer = new cloudflare.WorkerScript("producer", { + accountId, name: "producer", content: code, + queueBindings: [{name: "MY_QUEUE", queue: queue.id}], +}); + +// Consumer +const consumer = new cloudflare.WorkerScript("consumer", { + accountId, name: "consumer", content: code, + queueConsumers: [{queue: queue.name, maxBatchSize: 10, maxRetries: 3}], +}); +``` + +## Pages Projects (cloudflare.PagesProject) + +```typescript +const pages = new cloudflare.PagesProject("my-site", { + accountId, name: "my-site", productionBranch: "main", + buildConfig: {buildCommand: "npm run build", destinationDir: "dist"}, + source: { + type: "github", + config: {owner: "my-org", repoName: "my-repo", productionBranch: "main"}, + }, + deploymentConfigs: { + production: { + environmentVariables: {NODE_VERSION: "18"}, + kvNamespaces: {MY_KV: kv.id}, + d1Databases: {DB: db.id}, + }, + }, +}); +``` + +## DNS Records (cloudflare.DnsRecord) + +```typescript +const zone = cloudflare.getZone({name: "example.com"}); +const record = new cloudflare.DnsRecord("www", { + zoneId: zone.then(z => z.id), name: "www", type: "A", + content: "192.0.2.1", ttl: 3600, proxied: true, +}); +``` + +## Workers Domains/Routes + +```typescript +// Route (pattern-based) +const route = new cloudflare.WorkerRoute("my-route", { + zoneId: zoneId, + pattern: "example.com/api/*", + scriptName: worker.name, +}); + +// Domain (dedicated subdomain) +const domain = new cloudflare.WorkersDomain("my-domain", { + accountId: accountId, + hostname: "api.example.com", + service: worker.name, + zoneId: zoneId, +}); +``` + +## Assets Configuration (v6.x) + +Serve static assets from Workers: + +```typescript +const worker = new cloudflare.WorkerScript("app", { + accountId: accountId, + name: "my-app", + content: code, + assets: { + path: "./public", // Local directory + // Assets uploaded and served from Workers + }, +}); +``` + +## v6.x Versioned Deployments (Advanced) + +For gradual rollouts, use 3-resource pattern: + +```typescript +// 1. Worker (container for versions) +const worker = new cloudflare.Worker("api", { + accountId: accountId, + name: "api-worker", +}); + +// 2. Version (immutable code + config) +const version = new cloudflare.WorkerVersion("v1", { + accountId: accountId, + workerId: worker.id, + content: fs.readFileSync("./dist/worker.js", "utf8"), + compatibilityDate: "2025-01-01", + compatibilityFlags: ["nodejs_compat"], + // Note: Bindings configured at deployment level +}); + +// 3. Deployment (version + bindings + traffic split) +const deployment = new cloudflare.WorkersDeployment("prod", { + accountId: accountId, + workerId: worker.id, + versionId: version.id, + // Bindings applied to deployment + kvNamespaceBindings: [{name: "MY_KV", namespaceId: kv.id}], +}); +``` + +**When to use:** Blue-green deployments, canary releases, gradual rollouts +**When NOT to use:** Simple single-version deployments (use WorkerScript) + +--- +See: [README.md](./README.md), [api.md](./api.md), [patterns.md](./patterns.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/pulumi/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/pulumi/gotchas.md new file mode 100644 index 0000000..f01592a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pulumi/gotchas.md @@ -0,0 +1,181 @@ +# Troubleshooting & Best Practices + +## Common Errors + +### "No bundler/build step" - Pulumi uploads raw code + +**Problem:** Worker fails with "Cannot use import statement outside a module" +**Cause:** Pulumi doesn't bundle Worker code - uploads exactly what you provide +**Solution:** Build Worker BEFORE Pulumi deploy + +```typescript +// WRONG: Pulumi won't bundle this +const worker = new cloudflare.WorkerScript("worker", { + content: fs.readFileSync("./src/index.ts", "utf8"), // Raw TS file +}); + +// RIGHT: Build first, then deploy +import * as command from "@pulumi/command"; +const build = new command.local.Command("build", { + create: "npm run build", + dir: "./worker", +}); +const worker = new cloudflare.WorkerScript("worker", { + content: build.stdout.apply(() => fs.readFileSync("./worker/dist/index.js", "utf8")), +}, {dependsOn: [build]}); +``` + +### "wrangler.toml not consumed" - Config drift + +**Problem:** Local wrangler dev works, Pulumi deploy fails +**Cause:** Pulumi ignores wrangler.toml - must duplicate config +**Solution:** Generate wrangler.toml from Pulumi or keep synced manually + +```typescript +// Pattern: Export Pulumi config to wrangler.toml +const workerConfig = { + name: "my-worker", + compatibilityDate: "2025-01-01", + compatibilityFlags: ["nodejs_compat"], +}; + +new command.local.Command("generate-wrangler", { + create: pulumi.interpolate`cat > wrangler.toml <.yaml +config: + cloudflare:accountId: "abc123..." +``` + +### "Binding name mismatch" + +**Problem:** Worker fails with "env.MY_KV is undefined" +**Cause:** Binding name in Pulumi != name in Worker code +**Solution:** Match exactly (case-sensitive) + +```typescript +// Pulumi +kvNamespaceBindings: [{name: "MY_KV", namespaceId: kv.id}] + +// Worker code +export default { async fetch(request, env) { await env.MY_KV.get("key"); }} +``` + +### "API token permissions insufficient" + +**Problem:** `Error: authentication error (10000)` +**Cause:** Token lacks required permissions +**Solution:** Grant token permissions: Account.Workers Scripts:Edit, Account.Account Settings:Read + +### "Resource not found after import" + +**Problem:** Imported resource shows as changed on next `pulumi up` +**Cause:** State mismatch between actual resource and Pulumi config +**Solution:** Check property names/types match exactly + +```bash +pulumi import cloudflare:index/workerScript:WorkerScript my-worker / +pulumi preview # If shows changes, adjust Pulumi code to match actual resource +``` + +### "v6.x Worker versioning confusion" + +**Problem:** Worker deployed but not receiving traffic +**Cause:** v6.x requires Worker + WorkerVersion + WorkersDeployment (3 resources) +**Solution:** Use WorkerScript (auto-versioning) OR full versioning pattern + +```typescript +// SIMPLE: WorkerScript auto-versions (default behavior) +const worker = new cloudflare.WorkerScript("worker", { + accountId, name: "my-worker", content: code, +}); + +// ADVANCED: Manual versioning for gradual rollouts (v6.x) +const worker = new cloudflare.Worker("worker", {accountId, name: "my-worker"}); +const version = new cloudflare.WorkerVersion("v1", { + accountId, workerId: worker.id, content: code, compatibilityDate: "2025-01-01", +}); +const deployment = new cloudflare.WorkersDeployment("prod", { + accountId, workerId: worker.id, versionId: version.id, +}); +``` + +## Best Practices + +1. **Always set compatibilityDate** - Locks Worker behavior, prevents breaking changes +2. **Build before deploy** - Pulumi doesn't bundle; use Command resource or CI build step +3. **Match binding names** - Case-sensitive, must match between Pulumi and Worker code +4. **Use dependsOn for migrations** - Ensure D1 migrations run before Worker deploys +5. **Version Worker content** - Add VERSION binding to force redeployment on content changes +6. **Store secrets in stack config** - Use `pulumi config set --secret` for API keys + +## Limits + +| Resource | Limit | Notes | +|----------|-------|-------| +| Worker script size | 10 MB | Includes all dependencies, after compression | +| Worker CPU time | 50ms (free), 30s (paid) | Per request | +| KV keys per namespace | Unlimited | 1000 ops/sec write, 100k ops/sec read | +| R2 storage | Unlimited | Class A ops: 1M/mo free, Class B: 10M/mo free | +| D1 databases | 50,000 per account | Free: 10 per account, 5 GB each | +| Queues | 10,000 per account | Free: 1M ops/day | +| Pages projects | 500 per account | Free: 100 projects | +| API requests | Varies by plan | ~1200 req/5min on free | + +## Resources + +- **Pulumi Registry:** https://www.pulumi.com/registry/packages/cloudflare/ +- **API Docs:** https://www.pulumi.com/registry/packages/cloudflare/api-docs/ +- **GitHub:** https://github.com/pulumi/pulumi-cloudflare +- **Cloudflare Docs:** https://developers.cloudflare.com/ +- **Workers Docs:** https://developers.cloudflare.com/workers/ + +--- +See: [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/pulumi/patterns.md b/plugins/cloudflare/skills/cloudflare/references/pulumi/patterns.md new file mode 100644 index 0000000..c843d54 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/pulumi/patterns.md @@ -0,0 +1,191 @@ +# Architecture Patterns + +## Component Resources + +```typescript +class WorkerApp extends pulumi.ComponentResource { + constructor(name: string, args: WorkerAppArgs, opts?) { + super("custom:cloudflare:WorkerApp", name, {}, opts); + const defaultOpts = {parent: this}; + + this.kv = new cloudflare.WorkersKvNamespace(`${name}-kv`, {accountId: args.accountId, title: `${name}-kv`}, defaultOpts); + this.worker = new cloudflare.WorkerScript(`${name}-worker`, { + accountId: args.accountId, name: `${name}-worker`, content: args.workerCode, + module: true, kvNamespaceBindings: [{name: "KV", namespaceId: this.kv.id}], + }, defaultOpts); + this.domain = new cloudflare.WorkersDomain(`${name}-domain`, { + accountId: args.accountId, hostname: args.domain, service: this.worker.name, + }, defaultOpts); + } +} +``` + +## Full-Stack Worker App + +```typescript +const kv = new cloudflare.WorkersKvNamespace("cache", {accountId, title: "api-cache"}); +const db = new cloudflare.D1Database("db", {accountId, name: "app-database"}); +const bucket = new cloudflare.R2Bucket("assets", {accountId, name: "app-assets"}); + +const apiWorker = new cloudflare.WorkerScript("api", { + accountId, name: "api-worker", content: fs.readFileSync("./dist/api.js", "utf8"), + module: true, kvNamespaceBindings: [{name: "CACHE", namespaceId: kv.id}], + d1DatabaseBindings: [{name: "DB", databaseId: db.id}], + r2BucketBindings: [{name: "ASSETS", bucketName: bucket.name}], +}); +``` + +## Multi-Environment Setup + +```typescript +const stack = pulumi.getStack(); +const worker = new cloudflare.WorkerScript(`worker-${stack}`, { + accountId, name: `my-worker-${stack}`, content: code, + plainTextBindings: [{name: "ENVIRONMENT", text: stack}], +}); +``` + +## Queue-Based Processing + +```typescript +const queue = new cloudflare.Queue("processing-queue", {accountId, name: "image-processing"}); + +// Producer: API receives requests +const apiWorker = new cloudflare.WorkerScript("api", { + accountId, name: "api-worker", content: apiCode, + queueBindings: [{name: "PROCESSING_QUEUE", queue: queue.id}], +}); + +// Consumer: Process async +const processorWorker = new cloudflare.WorkerScript("processor", { + accountId, name: "processor-worker", content: processorCode, + queueConsumers: [{queue: queue.name, maxBatchSize: 10, maxRetries: 3, maxWaitTimeMs: 5000}], + r2BucketBindings: [{name: "OUTPUT_BUCKET", bucketName: outputBucket.name}], +}); +``` + +## Microservices with Service Bindings + +```typescript +const authWorker = new cloudflare.WorkerScript("auth", {accountId, name: "auth-service", content: authCode}); +const apiWorker = new cloudflare.WorkerScript("api", { + accountId, name: "api-service", content: apiCode, + serviceBindings: [{name: "AUTH", service: authWorker.name}], +}); +``` + +## Event-Driven Architecture + +```typescript +const eventQueue = new cloudflare.Queue("events", {accountId, name: "event-bus"}); +const producer = new cloudflare.WorkerScript("producer", { + accountId, name: "api-producer", content: producerCode, + queueBindings: [{name: "EVENTS", queue: eventQueue.id}], +}); +const consumer = new cloudflare.WorkerScript("consumer", { + accountId, name: "email-consumer", content: consumerCode, + queueConsumers: [{queue: eventQueue.name, maxBatchSize: 10}], +}); +``` + +## v6.x Versioned Deployments (Blue-Green/Canary) + +```typescript +const worker = new cloudflare.Worker("api", {accountId, name: "api-worker"}); +const v1 = new cloudflare.WorkerVersion("v1", {accountId, workerId: worker.id, content: fs.readFileSync("./dist/v1.js", "utf8"), compatibilityDate: "2025-01-01"}); +const v2 = new cloudflare.WorkerVersion("v2", {accountId, workerId: worker.id, content: fs.readFileSync("./dist/v2.js", "utf8"), compatibilityDate: "2025-01-01"}); + +// Gradual rollout: 10% v2, 90% v1 +const deployment = new cloudflare.WorkersDeployment("canary", { + accountId, workerId: worker.id, + versions: [{versionId: v2.id, percentage: 10}, {versionId: v1.id, percentage: 90}], + kvNamespaceBindings: [{name: "MY_KV", namespaceId: kv.id}], +}); +``` + +**Use:** Canary releases, A/B testing, blue-green. Most apps use `WorkerScript` (auto-versioning). + +## Wrangler.toml Generation (Bridge IaC with Local Dev) + +Generate wrangler.toml from Pulumi config to keep local dev in sync: + +```typescript +import * as command from "@pulumi/command"; + +const workerConfig = { + name: "my-worker", + compatibilityDate: "2025-01-01", + compatibilityFlags: ["nodejs_compat"], +}; + +// Create resources +const kv = new cloudflare.WorkersKvNamespace("kv", {accountId, title: "my-kv"}); +const db = new cloudflare.D1Database("db", {accountId, name: "my-db"}); +const bucket = new cloudflare.R2Bucket("bucket", {accountId, name: "my-bucket"}); + +// Generate wrangler.toml after resources created +const wranglerGen = new command.local.Command("gen-wrangler", { + create: pulumi.interpolate`cat > wrangler.toml < fs.readFileSync("./worker/dist/index.js", "utf8")), +}, {dependsOn: [build]}); +``` + +## Content SHA Pattern (Force Updates) + +Prevent false "no changes" detections: + +```typescript +const version = Date.now().toString(); +const worker = new cloudflare.WorkerScript("worker", { + accountId, name: "my-worker", content: code, + plainTextBindings: [{name: "VERSION", text: version}], // Forces deployment +}); +``` + +--- +See: [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/queues/README.md b/plugins/cloudflare/skills/cloudflare/references/queues/README.md new file mode 100644 index 0000000..5588fd2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/queues/README.md @@ -0,0 +1,96 @@ +# Cloudflare Queues + +Flexible message queuing for async task processing with guaranteed at-least-once delivery and configurable batching. + +## Overview + +Queues provide: +- At-least-once delivery guarantee +- Push-based (Worker) and pull-based (HTTP) consumers +- Configurable batching and retries +- Dead Letter Queues (DLQ) +- Delays up to 12 hours + +**Use cases:** Async processing, API buffering, rate limiting, event workflows, deferred jobs + +## Quick Start + +```bash +wrangler queues create my-queue +wrangler queues consumer add my-queue my-worker +``` + +```typescript +// Producer +await env.MY_QUEUE.send({ userId: 123, action: 'notify' }); + +// Consumer (with proper error handling) +export default { + async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + try { + await process(msg.body); + msg.ack(); + } catch (error) { + msg.retry({ delaySeconds: 60 }); + } + } + } +}; +``` + +## Critical Warnings + +**Before using Queues, understand these production mistakes:** + +1. **Uncaught errors retry ENTIRE batch** (not just failed message). Always use per-message try/catch. +2. **Messages not ack'd/retry'd will auto-retry forever** until max_retries. Always explicitly handle each message. + +See [gotchas.md](./gotchas.md) for detailed solutions. + +## Core Operations + +| Operation | Purpose | Limit | +|-----------|---------|-------| +| `send(body, options?)` | Publish message | 128 KB | +| `sendBatch(messages)` | Bulk publish | 100 msgs/256 KB | +| `message.ack()` | Acknowledge success | - | +| `message.retry(options?)` | Retry with delay | - | +| `batch.ackAll()` | Ack entire batch | - | + +## Architecture + +``` +[Producer Worker] → [Queue] → [Consumer Worker/HTTP] → [Processing] +``` + +- Max 10,000 queues per account +- 5,000 msgs/second per queue +- 4-14 day retention (configurable) + +## Reading Order + +**New to Queues?** Start here: +1. [configuration.md](./configuration.md) - Set up queues, bindings, consumers +2. [api.md](./api.md) - Send messages, handle batches, ack/retry patterns +3. [patterns.md](./patterns.md) - Real-world examples and integrations +4. [gotchas.md](./gotchas.md) - Critical warnings and troubleshooting + +**Task-based routing:** +- Setup queue → [configuration.md](./configuration.md) +- Send/receive messages → [api.md](./api.md) +- Implement specific pattern → [patterns.md](./patterns.md) +- Debug/troubleshoot → [gotchas.md](./gotchas.md) + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc setup, producer/consumer config, DLQ, content types +- [api.md](./api.md) - Send/batch methods, queue handler, ack/retry rules, type-safe patterns +- [patterns.md](./patterns.md) - Async tasks, buffering, rate limiting, D1/Workflows/DO integrations +- [gotchas.md](./gotchas.md) - Critical batch error handling, idempotency, error classification + +## See Also + +- [workers](../workers/) - Worker runtime for producers/consumers +- [r2](../r2/) - Process R2 event notifications via queues +- [d1](../d1/) - Batch write to D1 from queue consumers diff --git a/plugins/cloudflare/skills/cloudflare/references/queues/api.md b/plugins/cloudflare/skills/cloudflare/references/queues/api.md new file mode 100644 index 0000000..ded029c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/queues/api.md @@ -0,0 +1,206 @@ +# Queues API Reference + +## Producer: Send Messages + +```typescript +// Basic send +await env.MY_QUEUE.send({ url: request.url, timestamp: Date.now() }); + +// Options: delay (max 43200s), contentType (json|text|bytes|v8) +await env.MY_QUEUE.send(message, { delaySeconds: 600 }); +await env.MY_QUEUE.send(message, { delaySeconds: 0 }); // Override queue default + +// Batch (up to 100 msgs or 256 KB) +await env.MY_QUEUE.sendBatch([ + { body: 'msg1' }, + { body: 'msg2' }, + { body: 'msg3', options: { delaySeconds: 300 } } +]); + +// Non-blocking with ctx.waitUntil - send continues after response +ctx.waitUntil(env.MY_QUEUE.send({ data: 'async' })); + +// Background tasks in queue consumer +export default { + async queue(batch: MessageBatch, env: Env, ctx: ExecutionContext): Promise { + for (const msg of batch.messages) { + await processMessage(msg.body); + + // Fire-and-forget analytics (doesn't block ack) + ctx.waitUntil( + env.ANALYTICS_QUEUE.send({ messageId: msg.id, processedAt: Date.now() }) + ); + + msg.ack(); + } + } +}; +``` + +## Consumer: Push-based (Worker) + +```typescript +// Type-safe handler with ExportedHandler +interface Env { + MY_QUEUE: Queue; + DB: D1Database; +} + +export default { + async queue(batch: MessageBatch, env: Env, ctx: ExecutionContext): Promise { + // batch.queue, batch.messages.length + for (const msg of batch.messages) { + // msg.id, msg.body, msg.timestamp, msg.attempts + try { + await processMessage(msg.body); + msg.ack(); + } catch (error) { + msg.retry({ delaySeconds: 600 }); + } + } + } +} satisfies ExportedHandler; +``` + +**CRITICAL WARNINGS:** + +1. **Messages not explicitly ack'd or retry'd will auto-retry indefinitely** until `max_retries` is reached. Always call `msg.ack()` or `msg.retry()` for each message. + +2. **Throwing uncaught errors retries the ENTIRE batch**, not just the failed message. Always wrap individual message processing in try/catch and call `msg.retry()` explicitly per message. + +```typescript +// ❌ BAD: Uncaught error retries entire batch +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + await riskyOperation(msg.body); // If this throws, entire batch retries + msg.ack(); + } +} + +// ✅ GOOD: Catch per message, handle individually +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + try { + await riskyOperation(msg.body); + msg.ack(); + } catch (error) { + msg.retry({ delaySeconds: 60 }); + } + } +} +``` + +## Ack/Retry Precedence Rules + +1. **Per-message calls take precedence**: If you call both `msg.ack()` and `msg.retry()`, last call wins +2. **Batch calls don't override**: `batch.ackAll()` only affects messages without explicit ack/retry +3. **No action = automatic retry**: Messages with no explicit action retry with configured delay + +```typescript +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + msg.ack(); // Message marked for ack + msg.retry(); // Overrides ack - message will retry + } + + batch.ackAll(); // Only affects messages not explicitly handled above +} +``` + +## Batch Operations + +```typescript +// Acknowledge entire batch +try { + await bulkProcess(batch.messages); + batch.ackAll(); +} catch (error) { + batch.retryAll({ delaySeconds: 300 }); +} +``` + +## Exponential Backoff + +```typescript +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + try { + await processMessage(msg.body); + msg.ack(); + } catch (error) { + // 30s, 60s, 120s, 240s, 480s, ... up to 12h max + const delay = Math.min(30 * (2 ** msg.attempts), 43200); + msg.retry({ delaySeconds: delay }); + } + } +} +``` + +## Multiple Queues, Single Consumer + +```typescript +export default { + async queue(batch: MessageBatch, env: Env): Promise { + switch (batch.queue) { + case 'high-priority': await processUrgent(batch.messages); break; + case 'low-priority': await processDeferred(batch.messages); break; + case 'email': await sendEmails(batch.messages); break; + default: batch.retryAll(); + } + } +}; +``` + +## Consumer: Pull-based (HTTP) + +```typescript +// Pull messages +const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/messages/pull`, + { + method: 'POST', + headers: { 'authorization': `Bearer ${API_TOKEN}`, 'content-type': 'application/json' }, + body: JSON.stringify({ visibility_timeout_ms: 6000, batch_size: 50 }) + } +); + +const data = await response.json(); + +// Acknowledge +await fetch( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/messages/ack`, + { + method: 'POST', + headers: { 'authorization': `Bearer ${API_TOKEN}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + acks: [{ lease_id: msg.lease_id }], + retries: [{ lease_id: msg2.lease_id, delay_seconds: 600 }] + }) + } +); +``` + +## Interfaces + +```typescript +interface MessageBatch { + readonly queue: string; + readonly messages: Message[]; + ackAll(): void; + retryAll(options?: QueueRetryOptions): void; +} + +interface Message { + readonly id: string; + readonly timestamp: Date; + readonly body: Body; + readonly attempts: number; + ack(): void; + retry(options?: QueueRetryOptions): void; +} + +interface QueueSendOptions { + contentType?: 'text' | 'bytes' | 'json' | 'v8'; + delaySeconds?: number; // 0-43200 +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/queues/configuration.md b/plugins/cloudflare/skills/cloudflare/references/queues/configuration.md new file mode 100644 index 0000000..e6c629f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/queues/configuration.md @@ -0,0 +1,144 @@ +# Queues Configuration + +## Create Queue + +```bash +wrangler queues create my-queue +wrangler queues create my-queue --retention-period-hours=336 # 14 days +wrangler queues create my-queue --delivery-delay-secs=300 +``` + +## Producer Binding + +**wrangler.jsonc:** +```jsonc +{ + "queues": { + "producers": [ + { + "queue": "my-queue-name", + "binding": "MY_QUEUE", + "delivery_delay": 60 // Optional: default delay in seconds + } + ] + } +} +``` + +## Consumer Configuration (Push-based) + +**wrangler.jsonc:** +```jsonc +{ + "queues": { + "consumers": [ + { + "queue": "my-queue-name", + "max_batch_size": 10, // 1-100, default 10 + "max_batch_timeout": 5, // 0-60s, default 5 + "max_retries": 3, // default 3, max 100 + "dead_letter_queue": "my-dlq", // optional + "retry_delay": 300 // optional: delay retries in seconds + } + ] + } +} +``` + +## Consumer Configuration (Pull-based) + +**wrangler.jsonc:** +```jsonc +{ + "queues": { + "consumers": [ + { + "queue": "my-queue-name", + "type": "http_pull", + "visibility_timeout_ms": 5000, // default 30000, max 12h + "max_retries": 5, + "dead_letter_queue": "my-dlq" + } + ] + } +} +``` + +## TypeScript Types + +```typescript +interface Env { + MY_QUEUE: Queue; + ANALYTICS_QUEUE: Queue; +} + +interface MessageBody { + id: string; + action: 'create' | 'update' | 'delete'; + data: Record; +} + +export default { + async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + console.log(msg.body.action); + msg.ack(); + } + } +} satisfies ExportedHandler; +``` + +## Content Type Selection + +Choose content type based on consumer type and data requirements: + +| Content Type | Use When | Readable By | Supports | Size | +|--------------|----------|-------------|----------|------| +| `json` | Pull consumers, dashboard visibility, simple objects | All (push/pull/dashboard) | JSON-serializable types only | Medium | +| `v8` | Push consumers only, complex JS objects | Push consumers only | Date, Map, Set, BigInt, typed arrays | Small | +| `text` | String-only payloads | All | Strings only | Smallest | +| `bytes` | Binary data (images, files) | All | ArrayBuffer, Uint8Array | Variable | + +**Decision tree:** +1. Need to view in dashboard or use pull consumer? → Use `json` +2. Need Date, Map, Set, or other V8 types? → Use `v8` (push consumers only) +3. Just strings? → Use `text` +4. Binary data? → Use `bytes` + +```typescript +// JSON: Good for simple objects, pull consumers, dashboard visibility +await env.QUEUE.send({ id: 123, name: 'test' }, { contentType: 'json' }); + +// V8: Good for Date, Map, Set (push consumers only) +await env.QUEUE.send({ + created: new Date(), + tags: new Set(['a', 'b']) +}, { contentType: 'v8' }); + +// Text: Simple strings +await env.QUEUE.send('process-user-123', { contentType: 'text' }); + +// Bytes: Binary data +await env.QUEUE.send(imageBuffer, { contentType: 'bytes' }); +``` + +**Default behavior:** If not specified, Cloudflare auto-selects `json` for JSON-serializable objects and `v8` for complex types. + +**IMPORTANT:** `v8` messages cannot be read by pull consumers or viewed in the dashboard. Use `json` if you need visibility or pull-based consumption. + +## CLI Commands + +```bash +# Consumer management +wrangler queues consumer add my-queue my-worker --batch-size=50 --max-retries=5 +wrangler queues consumer http add my-queue +wrangler queues consumer worker remove my-queue my-worker +wrangler queues consumer http remove my-queue + +# Queue operations +wrangler queues list +wrangler queues pause my-queue +wrangler queues resume my-queue +wrangler queues purge my-queue +wrangler queues delete my-queue +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/queues/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/queues/gotchas.md new file mode 100644 index 0000000..b93cbe2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/queues/gotchas.md @@ -0,0 +1,206 @@ +# Queues Gotchas & Troubleshooting + +## CRITICAL: Top Production Mistakes + +### 1. "Entire Batch Retried After Single Error" + +**Problem:** Throwing uncaught error in queue handler retries the entire batch, not just the failed message +**Cause:** Uncaught exceptions propagate to the runtime, triggering batch-level retry +**Solution:** Always wrap individual message processing in try/catch and call `msg.retry()` explicitly + +```typescript +// ❌ BAD: Throws error, retries entire batch +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + await riskyOperation(msg.body); // If this throws, entire batch retries + msg.ack(); + } +} + +// ✅ GOOD: Catch per message, handle individually +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + try { + await riskyOperation(msg.body); + msg.ack(); + } catch (error) { + msg.retry({ delaySeconds: 60 }); + } + } +} +``` + +### 2. "Messages Retry Forever" + +**Problem:** Messages not explicitly ack'd or retry'd will auto-retry indefinitely +**Cause:** Runtime default behavior retries unhandled messages until `max_retries` reached +**Solution:** Always call `msg.ack()` or `msg.retry()` for each message. Never leave messages unhandled. + +```typescript +// ❌ BAD: Skipped messages auto-retry forever +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + if (shouldProcess(msg.body)) { + await process(msg.body); + msg.ack(); + } + // Missing: msg.ack() for skipped messages - they will retry! + } +} + +// ✅ GOOD: Explicitly handle all messages +async queue(batch: MessageBatch): Promise { + for (const msg of batch.messages) { + if (shouldProcess(msg.body)) { + await process(msg.body); + msg.ack(); + } else { + msg.ack(); // Explicitly ack even if not processing + } + } +} +``` + +## Common Errors + +### "Duplicate Message Processing" + +**Problem:** Same message processed multiple times +**Cause:** At-least-once delivery guarantee means duplicates are possible during retries +**Solution:** Design consumers to be idempotent by tracking processed message IDs in KV with expiration TTL + +```typescript +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + const processed = await env.PROCESSED_KV.get(msg.id); + if (processed) { + msg.ack(); + continue; + } + + await processMessage(msg.body); + await env.PROCESSED_KV.put(msg.id, '1', { expirationTtl: 86400 }); + msg.ack(); + } +} +``` + +### "Pull Consumer Can't Decode Messages" + +**Problem:** Pull consumer or dashboard shows unreadable message bodies +**Cause:** Messages sent with `v8` content type are only decodable by Workers push consumers +**Solution:** Use `json` content type for pull consumers or dashboard visibility + +```typescript +// Use json for pull consumers +await env.MY_QUEUE.send(data, { contentType: 'json' }); + +// Use v8 only for push consumers with complex JS types +await env.MY_QUEUE.send({ date: new Date(), tags: new Set() }, { contentType: 'v8' }); +``` + +### "Messages Not Being Delivered" + +**Problem:** Messages sent but consumer not processing +**Cause:** Queue paused, consumer not configured, or consumer errors +**Solution:** Check queue status with `wrangler queues list`, verify consumer configured with `wrangler queues consumer add`, and check logs with `wrangler tail` + +### "High Dead Letter Queue Rate" + +**Problem:** Many messages ending up in DLQ +**Cause:** Consumer repeatedly failing to process messages after max retries +**Solution:** Review consumer error logs, check external dependency availability, verify message format matches expectations, or increase retry delay + +## Error Classification Patterns + +Classify errors to decide whether to retry or DLQ: + +```typescript +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + try { + await processMessage(msg.body); + msg.ack(); + } catch (error) { + // Transient errors: retry with backoff + if (isRetryable(error)) { + const delay = Math.min(30 * (2 ** msg.attempts), 43200); + msg.retry({ delaySeconds: delay }); + } + // Permanent errors: ack to avoid infinite retries + else { + console.error('Permanent error, sending to DLQ:', error); + await env.ERROR_LOG.put(msg.id, JSON.stringify({ msg: msg.body, error: String(error) })); + msg.ack(); // Prevent further retries + } + } + } +} + +function isRetryable(error: unknown): boolean { + if (error instanceof Response) { + // Retry: rate limits, timeouts, server errors + return error.status === 429 || error.status >= 500; + } + if (error instanceof Error) { + // Don't retry: validation, auth, not found + return !error.message.includes('validation') && + !error.message.includes('unauthorized') && + !error.message.includes('not found'); + } + return false; // Unknown errors don't retry +} +``` + +### "CPU Time Exceeded in Consumer" + +**Problem:** Consumer fails with CPU time limit exceeded +**Cause:** Consumer processing exceeding 30s default CPU time limit +**Solution:** Increase CPU limit in wrangler.jsonc: `{ "limits": { "cpu_ms": 300000 } }` (5 minutes max) + +## Content Type Decision Guide + +**When to use each content type:** + +| Content Type | Use When | Readable By | Supports | +|--------------|----------|-------------|----------| +| `json` (default) | Pull consumers, dashboard visibility, simple objects | All (push/pull/dashboard) | JSON-serializable types only | +| `v8` | Push consumers only, complex JS objects | Push consumers only | Date, Map, Set, BigInt, typed arrays | +| `text` | String-only payloads | All | Strings only | +| `bytes` | Binary data (images, files) | All | ArrayBuffer, Uint8Array | + +**Decision tree:** +1. Need to view in dashboard or use pull consumer? → Use `json` +2. Need Date, Map, Set, or other V8 types? → Use `v8` (push consumers only) +3. Just strings? → Use `text` +4. Binary data? → Use `bytes` + +```typescript +// Dashboard/pull: use json +await env.QUEUE.send({ id: 123, name: 'test' }, { contentType: 'json' }); + +// Complex JS types (push only): use v8 +await env.QUEUE.send({ + created: new Date(), + tags: new Set(['a', 'b']) +}, { contentType: 'v8' }); +``` + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Max queues | 10,000 | Per account | +| Message size | 128 KB | Maximum per message | +| Batch size (consumer) | 100 messages | Maximum messages per batch | +| Batch size (sendBatch) | 100 msgs or 256 KB | Whichever limit reached first | +| Throughput | 5,000 msgs/sec | Per queue | +| Retention | 4-14 days | Configurable retention period | +| Max backlog | 25 GB | Maximum queue backlog size | +| Max delay | 12 hours (43,200s) | Maximum message delay | +| Max retries | 100 | Maximum retry attempts | +| CPU time default | 30s | Per consumer invocation | +| CPU time max | 300s (5 min) | Configurable via `limits.cpu_ms` | +| Operations per message | 3 (write + read + delete) | Base cost per message | +| Pricing | $0.40 per 1M operations | After 1M free operations | +| Message charging | Per 64 KB chunk | Messages charged in 64 KB increments | diff --git a/plugins/cloudflare/skills/cloudflare/references/queues/patterns.md b/plugins/cloudflare/skills/cloudflare/references/queues/patterns.md new file mode 100644 index 0000000..9ff01c1 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/queues/patterns.md @@ -0,0 +1,220 @@ +# Queues Patterns & Best Practices + +## Async Task Processing + +```typescript +// Producer: Accept request, queue work +export default { + async fetch(request: Request, env: Env): Promise { + const { userId, reportType } = await request.json(); + await env.REPORT_QUEUE.send({ userId, reportType, requestedAt: Date.now() }); + return Response.json({ message: 'Report queued', status: 'pending' }); + } +}; + +// Consumer: Process reports +export default { + async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + const { userId, reportType } = msg.body; + const report = await generateReport(userId, reportType, env); + await env.REPORTS_BUCKET.put(`${userId}/${reportType}.pdf`, report); + msg.ack(); + } + } +}; +``` + +## Buffering API Calls + +```typescript +// Producer: Queue log entries +ctx.waitUntil(env.LOGS_QUEUE.send({ + method: request.method, + url: request.url, + timestamp: Date.now() +})); + +// Consumer: Batch write to external API +async queue(batch: MessageBatch, env: Env): Promise { + const logs = batch.messages.map(m => m.body); + await fetch(env.LOG_ENDPOINT, { method: 'POST', body: JSON.stringify({ logs }) }); + batch.ackAll(); +} +``` + +## Rate Limiting Upstream + +```typescript +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + try { + await callRateLimitedAPI(msg.body); + msg.ack(); + } catch (error) { + if (error.status === 429) { + const retryAfter = parseInt(error.headers.get('Retry-After') || '60'); + msg.retry({ delaySeconds: retryAfter }); + } else throw error; + } + } +} +``` + +## Event-Driven Workflows + +```typescript +// R2 event → Queue → Worker +export default { + async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + const event = msg.body; + if (event.action === 'PutObject') { + await processNewFile(event.object.key, env); + } else if (event.action === 'DeleteObject') { + await cleanupReferences(event.object.key, env); + } + msg.ack(); + } + } +}; +``` + +## Dead Letter Queue Pattern + +```typescript +// Main queue: After max_retries, goes to DLQ automatically +export default { + async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + try { + await riskyOperation(msg.body); + msg.ack(); + } catch (error) { + console.error(`Failed after ${msg.attempts} attempts:`, error); + } + } + } +}; + +// DLQ consumer: Log and store failed messages +export default { + async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + await env.FAILED_KV.put(msg.id, JSON.stringify(msg.body)); + msg.ack(); + } + } +}; +``` + +## Priority Queues + +High priority: `max_batch_size: 5, max_batch_timeout: 1`. Low priority: `max_batch_size: 100, max_batch_timeout: 30`. + +## Delayed Job Processing + +```typescript +await env.EMAIL_QUEUE.send({ to, template, userId }, { delaySeconds: 3600 }); +``` + +## Fan-out Pattern + +```typescript +async fetch(request: Request, env: Env): Promise { + const event = await request.json(); + + // Send to multiple queues for parallel processing + await Promise.all([ + env.ANALYTICS_QUEUE.send(event), + env.NOTIFICATIONS_QUEUE.send(event), + env.AUDIT_LOG_QUEUE.send(event) + ]); + + return Response.json({ status: 'processed' }); +} +``` + +## Idempotency Pattern + +```typescript +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + // Check if already processed + const processed = await env.PROCESSED_KV.get(msg.id); + if (processed) { + msg.ack(); + continue; + } + + await processMessage(msg.body); + await env.PROCESSED_KV.put(msg.id, '1', { expirationTtl: 86400 }); + msg.ack(); + } +} +``` + +## Integration: D1 Batch Writes + +```typescript +async queue(batch: MessageBatch, env: Env): Promise { + // Collect all inserts for single D1 batch + const statements = batch.messages.map(msg => + env.DB.prepare('INSERT INTO events (id, data, created) VALUES (?, ?, ?)') + .bind(msg.id, JSON.stringify(msg.body), Date.now()) + ); + + try { + await env.DB.batch(statements); + batch.ackAll(); + } catch (error) { + console.error('D1 batch failed:', error); + batch.retryAll({ delaySeconds: 60 }); + } +} +``` + +## Integration: Workflows + +```typescript +// Queue triggers Workflow for long-running tasks +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + try { + const instance = await env.MY_WORKFLOW.create({ + id: msg.id, + params: msg.body + }); + console.log('Workflow started:', instance.id); + msg.ack(); + } catch (error) { + msg.retry({ delaySeconds: 30 }); + } + } +} +``` + +## Integration: Durable Objects + +```typescript +// Queue distributes work to Durable Objects by ID +async queue(batch: MessageBatch, env: Env): Promise { + for (const msg of batch.messages) { + const { userId, action } = msg.body; + + // Route to user-specific DO + const id = env.USER_DO.idFromName(userId); + const stub = env.USER_DO.get(id); + + try { + await stub.fetch(new Request('https://do/process', { + method: 'POST', + body: JSON.stringify({ action, messageId: msg.id }) + })); + msg.ack(); + } catch (error) { + msg.retry({ delaySeconds: 60 }); + } + } +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/README.md b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/README.md new file mode 100644 index 0000000..88702fa --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/README.md @@ -0,0 +1,149 @@ +# Cloudflare R2 Data Catalog Skill Reference + +Expert guidance for Cloudflare R2 Data Catalog - Apache Iceberg catalog built into R2 buckets. + +## Reading Order + +**New to R2 Data Catalog?** Start here: +1. Read "What is R2 Data Catalog?" and "When to Use" below +2. [configuration.md](configuration.md) - Enable catalog, create tokens +3. [patterns.md](patterns.md) - PyIceberg setup and common patterns +4. [api.md](api.md) - REST API reference as needed +5. [gotchas.md](gotchas.md) - Troubleshooting when issues arise + +**Quick reference?** Jump to: +- [Enable catalog on bucket](configuration.md#enable-catalog-on-bucket) +- [PyIceberg connection pattern](patterns.md#pyiceberg-connection-pattern) +- [Permission errors](gotchas.md#permission-errors) + +## What is R2 Data Catalog? + +R2 Data Catalog is a **managed Apache Iceberg REST catalog** built directly into R2 buckets. It provides: + +- **Apache Iceberg tables** - ACID transactions, schema evolution, time-travel queries +- **Zero-egress costs** - Query from any cloud/region without data transfer fees +- **Standard REST API** - Works with Spark, PyIceberg, Snowflake, Trino, DuckDB +- **No infrastructure** - Fully managed, no catalog servers to run +- **Public beta** - Available to all R2 subscribers, no extra cost beyond R2 storage + +### What is Apache Iceberg? + +Open table format for analytics datasets in object storage. Features: +- **ACID transactions** - Safe concurrent reads/writes +- **Metadata optimization** - Fast queries without full scans +- **Schema evolution** - Add/rename/delete columns without rewrites +- **Time-travel** - Query historical snapshots +- **Partitioning** - Organize data for efficient queries + +## When to Use + +**Use R2 Data Catalog for:** +- **Log analytics** - Store and query application/system logs +- **Data lakes/warehouses** - Analytical datasets queried by multiple engines +- **BI pipelines** - Aggregate data for dashboards and reports +- **Multi-cloud analytics** - Share data across clouds without egress fees +- **Time-series data** - Event streams, metrics, sensor data + +**Don't use for:** +- **Transactional workloads** - Use D1 or external database instead +- **Sub-second latency** - Iceberg optimized for batch/analytical queries +- **Small datasets (<1GB)** - Setup overhead not worth it +- **Unstructured data** - Store files directly in R2, not as Iceberg tables + +## Architecture + +``` +┌─────────────────────────────────────────────────┐ +│ Query Engines │ +│ (PyIceberg, Spark, Trino, Snowflake, DuckDB) │ +└────────────────┬────────────────────────────────┘ + │ + │ REST API (OAuth2 token) + ▼ +┌─────────────────────────────────────────────────┐ +│ R2 Data Catalog (Managed Iceberg REST Catalog)│ +│ • Namespace/table metadata │ +│ • Transaction coordination │ +│ • Snapshot management │ +└────────────────┬────────────────────────────────┘ + │ + │ Vended credentials + ▼ +┌─────────────────────────────────────────────────┐ +│ R2 Bucket Storage │ +│ • Parquet data files │ +│ • Metadata files │ +│ • Manifest files │ +└─────────────────────────────────────────────────┘ +``` + +**Key concepts:** +- **Catalog URI** - REST endpoint for catalog operations (e.g., `https://.r2.cloudflarestorage.com/iceberg/`) +- **Warehouse** - Logical grouping of tables (typically same as bucket name) +- **Namespace** - Schema/database containing tables (e.g., `logs`, `analytics`) +- **Table** - Iceberg table with schema, data files, snapshots +- **Vended credentials** - Temporary S3 credentials catalog provides for data access + +## Limits + +| Resource | Limit | Notes | +|----------|-------|-------| +| Namespaces per catalog | No hard limit | Organize tables logically | +| Tables per namespace | <10,000 recommended | Performance degrades beyond this | +| Files per table | <100,000 recommended | Run compaction regularly | +| Snapshots per table | Configurable retention | Expire >7 days old | +| Partitions per table | 100-1,000 optimal | Too many = slow metadata ops | +| Table size | Same as R2 bucket | 10GB-10TB+ common | +| API rate limits | Standard R2 API limits | Shared with R2 storage operations | +| Target file size | 128-512 MB | After compaction | + +## Current Status + +**Public Beta** (as of Jan 2026) +- Available to all R2 subscribers +- No extra cost beyond standard R2 storage/operations +- Production-ready, but breaking changes possible +- Supports: namespaces, tables, snapshots, compaction, time-travel, table maintenance + +## Decision Tree: Is R2 Data Catalog Right For You? + +``` +Start → Need analytics on object storage data? + │ + ├─ No → Use R2 directly for object storage + │ + └─ Yes → Dataset >1GB with structured schema? + │ + ├─ No → Too small, use R2 + ad-hoc queries + │ + └─ Yes → Need ACID transactions or schema evolution? + │ + ├─ No → Consider simpler solutions (Parquet on R2) + │ + └─ Yes → Need multi-cloud/multi-tool access? + │ + ├─ No → D1 or external DB may be simpler + │ + └─ Yes → ✅ Use R2 Data Catalog +``` + +**Quick check:** If you answer "yes" to all: +- Dataset >1GB and growing +- Structured/tabular data (logs, events, metrics) +- Multiple query tools or cloud environments +- Need versioning, schema changes, or concurrent access + +→ R2 Data Catalog is a good fit. + +## In This Reference + +- **[configuration.md](configuration.md)** - Enable catalog, create API tokens, connect clients +- **[api.md](api.md)** - REST endpoints, operations, maintenance +- **[patterns.md](patterns.md)** - PyIceberg examples, common use cases +- **[gotchas.md](gotchas.md)** - Troubleshooting, best practices, limitations + +## See Also + +- [Cloudflare R2 Data Catalog Docs](https://developers.cloudflare.com/r2/data-catalog/) +- [Apache Iceberg Docs](https://iceberg.apache.org/) +- [PyIceberg Docs](https://py.iceberg.apache.org/) diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/api.md b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/api.md new file mode 100644 index 0000000..3d57d4f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/api.md @@ -0,0 +1,199 @@ +# API Reference + +R2 Data Catalog exposes standard [Apache Iceberg REST Catalog API](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml). + +## Quick Reference + +**Most common operations:** + +| Task | PyIceberg Code | +|------|----------------| +| Connect | `RestCatalog(name="r2", warehouse=bucket, uri=uri, token=token)` | +| List namespaces | `catalog.list_namespaces()` | +| Create namespace | `catalog.create_namespace("logs")` | +| Create table | `catalog.create_table(("ns", "table"), schema=schema)` | +| Load table | `catalog.load_table(("ns", "table"))` | +| Append data | `table.append(pyarrow_table)` | +| Query data | `table.scan().to_pandas()` | +| Compact files | `table.rewrite_data_files(target_file_size_bytes=128*1024*1024)` | +| Expire snapshots | `table.expire_snapshots(older_than=timestamp_ms, retain_last=10)` | + +## REST Endpoints + +Base: `https://.r2.cloudflarestorage.com/iceberg/` + +| Operation | Method | Path | +|-----------|--------|------| +| Catalog config | GET | `/v1/config` | +| List namespaces | GET | `/v1/namespaces` | +| Create namespace | POST | `/v1/namespaces` | +| Delete namespace | DELETE | `/v1/namespaces/{ns}` | +| List tables | GET | `/v1/namespaces/{ns}/tables` | +| Create table | POST | `/v1/namespaces/{ns}/tables` | +| Load table | GET | `/v1/namespaces/{ns}/tables/{table}` | +| Update table | POST | `/v1/namespaces/{ns}/tables/{table}` | +| Delete table | DELETE | `/v1/namespaces/{ns}/tables/{table}` | +| Rename table | POST | `/v1/tables/rename` | + +**Authentication:** Bearer token in header: `Authorization: Bearer ` + +## PyIceberg Client API + +Most users use PyIceberg, not raw REST. + +### Connection + +```python +from pyiceberg.catalog.rest import RestCatalog + +catalog = RestCatalog( + name="my_catalog", + warehouse="", + uri="", + token="", +) +``` + +### Namespace Operations + +```python +from pyiceberg.exceptions import NamespaceAlreadyExistsError + +namespaces = catalog.list_namespaces() # [('default',), ('logs',)] +catalog.create_namespace("logs", properties={"owner": "team"}) +catalog.drop_namespace("logs") # Must be empty +``` + +### Table Operations + +```python +from pyiceberg.schema import Schema +from pyiceberg.types import NestedField, StringType, IntegerType + +schema = Schema( + NestedField(1, "id", IntegerType(), required=True), + NestedField(2, "name", StringType(), required=False), +) +table = catalog.create_table(("logs", "app_logs"), schema=schema) +tables = catalog.list_tables("logs") +table = catalog.load_table(("logs", "app_logs")) +catalog.rename_table(("logs", "old"), ("logs", "new")) +``` + +### Data Operations + +```python +import pyarrow as pa + +data = pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}) +table.append(data) +table.overwrite(data) + +# Read with filters +scan = table.scan(row_filter="id > 100", selected_fields=["id", "name"]) +df = scan.to_pandas() +``` + +### Schema Evolution + +```python +from pyiceberg.types import IntegerType, LongType + +with table.update_schema() as update: + update.add_column("user_id", IntegerType(), doc="User ID") + update.rename_column("msg", "message") + update.delete_column("old_field") + update.update_column("id", field_type=LongType()) # int→long only +``` + +### Time-Travel + +```python +from datetime import datetime, timedelta + +# Query specific snapshot or timestamp +scan = table.scan(snapshot_id=table.snapshots()[-2].snapshot_id) +yesterday_ms = int((datetime.now() - timedelta(days=1)).timestamp() * 1000) +scan = table.scan(as_of_timestamp=yesterday_ms) +``` + +### Partitioning + +```python +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform +from pyiceberg.types import TimestampType + +partition_spec = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name="day") +) +table = catalog.create_table(("events", "actions"), schema=schema, partition_spec=partition_spec) +scan = table.scan(row_filter="day = '2026-01-27'") # Prunes partitions +``` + +## Table Maintenance + +### Compaction + +```python +files = table.scan().plan_files() +avg_mb = sum(f.file_size_in_bytes for f in files) / len(files) / (1024**2) +print(f"Files: {len(files)}, Avg: {avg_mb:.1f} MB") + +table.rewrite_data_files(target_file_size_bytes=128 * 1024 * 1024) +``` + +**When:** Avg <10MB or >1000 files. **Frequency:** High-write daily, medium weekly. + +### Snapshot Expiration + +```python +from datetime import datetime, timedelta + +seven_days_ms = int((datetime.now() - timedelta(days=7)).timestamp() * 1000) +table.expire_snapshots(older_than=seven_days_ms, retain_last=10) +``` + +**Retention:** Production 7-30d, dev 1-7d, audit 90+d. + +### Orphan Cleanup + +```python +three_days_ms = int((datetime.now() - timedelta(days=3)).timestamp() * 1000) +table.delete_orphan_files(older_than=three_days_ms) +``` + +⚠️ Always expire snapshots first, use 3+ day threshold, run during low traffic. + +### Full Maintenance + +```python +# Compact → Expire → Cleanup (in order) +if len(table.scan().plan_files()) > 1000: + table.rewrite_data_files(target_file_size_bytes=128 * 1024 * 1024) +seven_days_ms = int((datetime.now() - timedelta(days=7)).timestamp() * 1000) +table.expire_snapshots(older_than=seven_days_ms, retain_last=10) +three_days_ms = int((datetime.now() - timedelta(days=3)).timestamp() * 1000) +table.delete_orphan_files(older_than=three_days_ms) +``` + +## Metadata Inspection + +```python +table = catalog.load_table(("logs", "app_logs")) +print(table.schema()) +print(table.current_snapshot()) +print(table.properties) +print(f"Files: {len(table.scan().plan_files())}") +``` + +## Error Codes + +| Code | Meaning | Common Causes | +|------|---------|---------------| +| 401 | Unauthorized | Invalid/missing token | +| 404 | Not Found | Catalog not enabled, namespace/table missing | +| 409 | Conflict | Already exists, concurrent update | +| 422 | Validation | Invalid schema, incompatible type | + +See [gotchas.md](gotchas.md) for detailed troubleshooting. diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/configuration.md b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/configuration.md new file mode 100644 index 0000000..15915da --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/configuration.md @@ -0,0 +1,198 @@ +# Configuration + +How to enable R2 Data Catalog and configure authentication. + +## Prerequisites + +- Cloudflare account with [R2 subscription](https://developers.cloudflare.com/r2/pricing/) +- R2 bucket created +- Access to Cloudflare dashboard or Wrangler CLI + +## Enable Catalog on Bucket + +Choose one method: + +### Via Wrangler (Recommended) + +```bash +npx wrangler r2 bucket catalog enable +``` + +**Output:** +``` +✅ Data Catalog enabled for bucket 'my-bucket' + Catalog URI: https://.r2.cloudflarestorage.com/iceberg/my-bucket + Warehouse: my-bucket +``` + +### Via Dashboard + +1. Navigate to **R2** → Select your bucket → **Settings** tab +2. Scroll to "R2 Data Catalog" section → Click **Enable** +3. Note the **Catalog URI** and **Warehouse name** shown + +**Result:** +- Catalog URI: `https://.r2.cloudflarestorage.com/iceberg/` +- Warehouse: `` (same as bucket name) + +### Via API (Programmatic) + +```bash +curl -X POST \ + "https://api.cloudflare.com/client/v4/accounts//r2/buckets//catalog" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" +``` + +**Response:** +```json +{ + "result": { + "catalog_uri": "https://.r2.cloudflarestorage.com/iceberg/", + "warehouse": "" + }, + "success": true +} +``` + +## Check Catalog Status + +```bash +npx wrangler r2 bucket catalog status +``` + +**Output:** +``` +Catalog Status: enabled +Catalog URI: https://.r2.cloudflarestorage.com/iceberg/my-bucket +Warehouse: my-bucket +``` + +## Disable Catalog (If Needed) + +```bash +npx wrangler r2 bucket catalog disable +``` + +⚠️ **Warning:** Disabling does NOT delete tables/data. Files remain in bucket. Metadata becomes inaccessible until re-enabled. + +## API Token Creation + +R2 Data Catalog requires API token with **both** R2 Storage + R2 Data Catalog permissions. + +### Dashboard Method (Recommended) + +1. Go to **R2** → **Manage R2 API Tokens** → **Create API Token** +2. Select permission level: + - **Admin Read & Write** - Full catalog + storage access (read/write) + - **Admin Read only** - Read-only access (for query engines) +3. Copy token value immediately (shown only once) + +**Permission groups included:** +- `Workers R2 Data Catalog Write` (or Read) +- `Workers R2 Storage Bucket Item Write` (or Read) + +### API Method (Programmatic) + +Use Cloudflare API to create tokens programmatically. Required permissions: +- `Workers R2 Data Catalog Write` (or Read) +- `Workers R2 Storage Bucket Item Write` (or Read) + +## Client Configuration + +### PyIceberg + +```python +from pyiceberg.catalog.rest import RestCatalog + +catalog = RestCatalog( + name="my_catalog", + warehouse="", # Same as bucket name + uri="", # From enable command + token="", # From token creation +) +``` + +**Full example with credentials:** +```python +import os +from pyiceberg.catalog.rest import RestCatalog + +# Store credentials in environment variables +WAREHOUSE = os.getenv("R2_WAREHOUSE") # e.g., "my-bucket" +CATALOG_URI = os.getenv("R2_CATALOG_URI") # e.g., "https://abc123.r2.cloudflarestorage.com/iceberg/my-bucket" +TOKEN = os.getenv("R2_TOKEN") # API token + +catalog = RestCatalog( + name="r2_catalog", + warehouse=WAREHOUSE, + uri=CATALOG_URI, + token=TOKEN, +) + +# Test connection +print(catalog.list_namespaces()) +``` + +### Spark / Trino / DuckDB + +See [patterns.md](patterns.md) for integration examples with other query engines. + +## Connection String Format + +For quick reference: + +``` +Catalog URI: https://.r2.cloudflarestorage.com/iceberg/ +Warehouse: +Token: +``` + +**Where to find values:** + +| Value | Source | +|-------|--------| +| `` | Dashboard URL or `wrangler whoami` | +| `` | R2 bucket name | +| Catalog URI | Output from `wrangler r2 bucket catalog enable` | +| Token | R2 API Token creation page | + +## Security Best Practices + +1. **Store tokens securely** - Use environment variables or secret managers, never hardcode +2. **Use least privilege** - Read-only tokens for query engines, write tokens only where needed +3. **Rotate tokens regularly** - Create new tokens, test, then revoke old ones +4. **One token per application** - Easier to track and revoke if compromised +5. **Monitor token usage** - Check R2 analytics for unexpected patterns +6. **Bucket-scoped tokens** - Create tokens per bucket, not account-wide + +## Environment Variables Pattern + +```bash +# .env (never commit) +R2_CATALOG_URI=https://.r2.cloudflarestorage.com/iceberg/ +R2_WAREHOUSE= +R2_TOKEN= +``` + +```python +import os +from pyiceberg.catalog.rest import RestCatalog + +catalog = RestCatalog( + name="r2", + uri=os.getenv("R2_CATALOG_URI"), + warehouse=os.getenv("R2_WAREHOUSE"), + token=os.getenv("R2_TOKEN"), +) +``` + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| 404 "catalog not found" | Run `wrangler r2 bucket catalog enable ` | +| 401 "unauthorized" | Check token has both Catalog + Storage permissions | +| 403 on data files | Token needs both permission groups | + +See [gotchas.md](gotchas.md) for detailed troubleshooting. diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/gotchas.md new file mode 100644 index 0000000..6bfad9e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/gotchas.md @@ -0,0 +1,170 @@ +# Gotchas & Troubleshooting + +Common problems → causes → solutions. + +## Permission Errors + +### 401 Unauthorized + +**Error:** `"401 Unauthorized"` +**Cause:** Token missing R2 Data Catalog permissions. +**Solution:** Use "Admin Read & Write" token (includes catalog + storage permissions). Test with `catalog.list_namespaces()`. + +### 403 Forbidden + +**Error:** `"403 Forbidden"` on data files +**Cause:** Token lacks storage permissions. +**Solution:** Token needs both R2 Data Catalog + R2 Storage Bucket Item permissions. + +### Token Rotation Issues + +**Error:** New token fails after rotation. +**Solution:** Create new token → test in staging → update prod → monitor 24h → revoke old. + +## Catalog URI Issues + +### 404 Not Found + +**Error:** `"404 Catalog not found"` +**Cause:** Catalog not enabled or wrong URI. +**Solution:** Run `wrangler r2 bucket catalog enable `. URI must be HTTPS with `/iceberg/` and case-sensitive bucket name. + +### Wrong Warehouse + +**Error:** Cannot create/load tables. +**Cause:** Warehouse ≠ bucket name. +**Solution:** Set `warehouse="bucket-name"` to match bucket exactly. + +## Table and Schema Issues + +### Table/Namespace Already Exists + +**Error:** `"TableAlreadyExistsError"` +**Solution:** Use try/except to load existing or check first. + +### Namespace Not Found + +**Error:** Cannot create table. +**Solution:** Create namespace first: `catalog.create_namespace("ns")` + +### Schema Evolution Errors + +**Error:** `"422 Validation"` on schema update. +**Cause:** Incompatible change (required field, type shrink). +**Solution:** Only add nullable columns, compatible type widening (int→long, float→double). + +## Data and Query Issues + +### Empty Scan Results + +**Error:** Scan returns no data. +**Cause:** Incorrect filter or partition column. +**Solution:** Test without filter first: `table.scan().to_pandas()`. Verify partition column names. + +### Slow Queries + +**Error:** Performance degrades over time. +**Cause:** Too many small files. +**Solution:** Check file count, compact if >1000 or avg <10MB. See [api.md](api.md#compaction). + +### Type Mismatch + +**Error:** `"Cannot cast"` on append. +**Cause:** PyArrow types don't match Iceberg schema. +**Solution:** Cast to int64 (Iceberg default), not int32. Check `table.schema()`. + +## Compaction Issues + +### Compaction Issues + +**Problem:** File count unchanged or compaction takes hours. +**Cause:** Target size too large, or table too big for PyIceberg. +**Solution:** Only compact if avg <50MB. For >1TB tables, use Spark. Run during low-traffic periods. + +## Maintenance Issues + +### Snapshot/Orphan Issues + +**Problem:** Expiration fails or orphan cleanup deletes active data. +**Cause:** Too aggressive retention or wrong order. +**Solution:** Always expire snapshots first with `retain_last=10`, then cleanup orphans with 3+ day threshold. + +## Concurrency Issues + +### Concurrent Write Conflicts + +**Problem:** `CommitFailedException` with multiple writers. +**Cause:** Optimistic locking - simultaneous commits. +**Solution:** Add retry with exponential backoff (see [patterns.md](patterns.md#pattern-6-concurrent-writes-with-retry)). + +### Stale Metadata + +**Problem:** Old schema/data after external update. +**Cause:** Cached metadata. +**Solution:** Reload table: `table = catalog.load_table(("ns", "table"))` + +## Performance Optimization + +### Performance Tips + +**Scans:** Use `row_filter` and `selected_fields` to reduce data scanned. +**Partitions:** 100-1000 optimal. Avoid high cardinality (millions) or low (<10). +**Files:** Keep 100-500MB avg. Compact if <10MB or >10k files. + +## Limits + +| Resource | Recommended | Impact if Exceeded | +|----------|-------------|-------------------| +| Tables/namespace | <10k | Slow list ops | +| Files/table | <100k | Slow query planning | +| Partitions/table | 100-1k | Metadata overhead | +| Snapshots/table | Expire >7d | Metadata bloat | + +## Common Error Messages Reference + +| Error Message | Likely Cause | Fix | +|---------------|--------------|-----| +| `401 Unauthorized` | Missing/invalid token | Check token has catalog+storage permissions | +| `403 Forbidden` | Token lacks storage permissions | Add R2 Storage Bucket Item permission | +| `404 Not Found` | Catalog not enabled or wrong URI | Run `wrangler r2 bucket catalog enable` | +| `409 Conflict` | Table/namespace already exists | Use try/except or load existing | +| `422 Unprocessable Entity` | Schema validation failed | Check type compatibility, required fields | +| `CommitFailedException` | Concurrent write conflict | Add retry logic with backoff | +| `NamespaceAlreadyExistsError` | Namespace exists | Use try/except or load existing | +| `NoSuchTableError` | Table doesn't exist | Check namespace+table name, create first | +| `TypeError: Cannot cast` | PyArrow type mismatch | Cast data to match Iceberg schema | + +## Debugging Checklist + +When things go wrong, check in order: + +1. ✅ **Catalog enabled:** `npx wrangler r2 bucket catalog status ` +2. ✅ **Token permissions:** Both R2 Data Catalog + R2 Storage in dashboard +3. ✅ **Connection test:** `catalog.list_namespaces()` succeeds +4. ✅ **URI format:** HTTPS, includes `/iceberg/`, correct bucket name +5. ✅ **Warehouse name:** Matches bucket name exactly +6. ✅ **Namespace exists:** Create before `create_table()` +7. ✅ **Enable debug logging:** `logging.basicConfig(level=logging.DEBUG)` +8. ✅ **PyIceberg version:** `pip install --upgrade pyiceberg` (≥0.5.0) +9. ✅ **File health:** Compact if >1000 files or avg <10MB +10. ✅ **Snapshot count:** Expire if >100 snapshots + +## Enable Debug Logging + +```python +import logging +logging.basicConfig(level=logging.DEBUG) +# Now operations show HTTP requests/responses +``` + +## Resources + +- [Cloudflare Community](https://community.cloudflare.com/c/developers/workers/40) +- [Cloudflare Discord](https://discord.cloudflare.com) - #r2 channel +- [PyIceberg GitHub](https://github.com/apache/iceberg-python/issues) +- [Apache Iceberg Slack](https://iceberg.apache.org/community/) + +## Next Steps + +- [patterns.md](patterns.md) - Working examples +- [api.md](api.md) - API reference diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/patterns.md b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/patterns.md new file mode 100644 index 0000000..b6b181f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-data-catalog/patterns.md @@ -0,0 +1,191 @@ +# Common Patterns + +Practical patterns for R2 Data Catalog with PyIceberg. + +## PyIceberg Connection + +```python +import os +from pyiceberg.catalog.rest import RestCatalog +from pyiceberg.exceptions import NamespaceAlreadyExistsError + +catalog = RestCatalog( + name="r2_catalog", + warehouse=os.getenv("R2_WAREHOUSE"), # bucket name + uri=os.getenv("R2_CATALOG_URI"), # catalog endpoint + token=os.getenv("R2_TOKEN"), # API token +) + +# Create namespace (idempotent) +try: + catalog.create_namespace("default") +except NamespaceAlreadyExistsError: + pass +``` + +## Pattern 1: Log Analytics Pipeline + +Ingest logs incrementally, query by time/level. + +```python +import pyarrow as pa +from datetime import datetime +from pyiceberg.schema import Schema +from pyiceberg.types import NestedField, TimestampType, StringType, IntegerType +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform + +# Create partitioned table (once) +schema = Schema( + NestedField(1, "timestamp", TimestampType(), required=True), + NestedField(2, "level", StringType(), required=True), + NestedField(3, "service", StringType(), required=True), + NestedField(4, "message", StringType(), required=False), +) + +partition_spec = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name="day") +) + +catalog.create_namespace("logs") +table = catalog.create_table(("logs", "app_logs"), schema=schema, partition_spec=partition_spec) + +# Append logs (incremental) +data = pa.table({ + "timestamp": [datetime(2026, 1, 27, 10, 30, 0)], + "level": ["ERROR"], + "service": ["auth-service"], + "message": ["Failed login"], +}) +table.append(data) + +# Query by time + level (leverages partitioning) +scan = table.scan(row_filter="level = 'ERROR' AND day = '2026-01-27'") +errors = scan.to_pandas() +``` + +## Pattern 2: Time-Travel Queries + +```python +from datetime import datetime, timedelta + +table = catalog.load_table(("logs", "app_logs")) + +# Query specific snapshot +snapshot_id = table.current_snapshot().snapshot_id +data = table.scan(snapshot_id=snapshot_id).to_pandas() + +# Query as of timestamp (yesterday) +yesterday_ms = int((datetime.now() - timedelta(days=1)).timestamp() * 1000) +data = table.scan(as_of_timestamp=yesterday_ms).to_pandas() +``` + +## Pattern 3: Schema Evolution + +```python +from pyiceberg.types import StringType + +table = catalog.load_table(("users", "profiles")) + +with table.update_schema() as update: + update.add_column("email", StringType(), required=False) + update.rename_column("name", "full_name") +# Old readers ignore new columns, new readers see nulls for old data +``` + +## Pattern 4: Partitioned Tables + +```python +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform, IdentityTransform + +# Partition by day + country +partition_spec = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name="day"), + PartitionField(source_id=2, field_id=1001, transform=IdentityTransform(), name="country"), +) +table = catalog.create_table(("events", "user_events"), schema=schema, partition_spec=partition_spec) + +# Queries prune partitions automatically +scan = table.scan(row_filter="country = 'US' AND day = '2026-01-27'") +``` + +## Pattern 5: Table Maintenance + +```python +from datetime import datetime, timedelta + +table = catalog.load_table(("logs", "app_logs")) + +# Compact → expire → cleanup (in order) +table.rewrite_data_files(target_file_size_bytes=128 * 1024 * 1024) +seven_days_ms = int((datetime.now() - timedelta(days=7)).timestamp() * 1000) +table.expire_snapshots(older_than=seven_days_ms, retain_last=10) +three_days_ms = int((datetime.now() - timedelta(days=3)).timestamp() * 1000) +table.delete_orphan_files(older_than=three_days_ms) +``` + +See [api.md](api.md#table-maintenance) for detailed parameters. + +## Pattern 6: Concurrent Writes with Retry + +```python +from pyiceberg.exceptions import CommitFailedException +import time + +def append_with_retry(table, data, max_retries=3): + for attempt in range(max_retries): + try: + table.append(data) + return + except CommitFailedException: + if attempt == max_retries - 1: + raise + time.sleep(2 ** attempt) +``` + +## Pattern 7: Upsert Simulation + +```python +import pandas as pd +import pyarrow as pa + +# Read → merge → overwrite (not atomic, use Spark MERGE INTO for production) +existing = table.scan().to_pandas() +new_data = pd.DataFrame({"id": [1, 3], "value": [100, 300]}) +merged = pd.concat([existing, new_data]).drop_duplicates(subset=["id"], keep="last") +table.overwrite(pa.Table.from_pandas(merged)) +``` + +## Pattern 8: DuckDB Integration + +```python +import duckdb + +arrow_table = table.scan().to_arrow() +con = duckdb.connect() +con.register("logs", arrow_table) +result = con.execute("SELECT level, COUNT(*) FROM logs GROUP BY level").fetchdf() +``` + +## Pattern 9: Monitor Table Health + +```python +files = table.scan().plan_files() +avg_mb = sum(f.file_size_in_bytes for f in files) / len(files) / (1024**2) +print(f"Files: {len(files)}, Avg: {avg_mb:.1f}MB, Snapshots: {len(table.snapshots())}") + +if avg_mb < 10 or len(files) > 1000: + print("⚠️ Needs compaction") +``` + +## Best Practices + +| Area | Guideline | +|------|-----------| +| **Partitioning** | Use day/hour for time-series; 100-1000 partitions; avoid high cardinality | +| **File sizes** | Target 128-512MB; compact when avg <10MB or >10k files | +| **Schema** | Add columns as nullable (`required=False`); batch changes | +| **Maintenance** | Compact high-write daily/weekly; expire snapshots 7-30d; cleanup orphans after | +| **Concurrency** | Reads automatic; writes to different partitions safe; retry same partition | +| **Performance** | Filter on partitions; select only needed columns; batch appends 100MB+ | diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-sql/README.md b/plugins/cloudflare/skills/cloudflare/references/r2-sql/README.md new file mode 100644 index 0000000..c59a161 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-sql/README.md @@ -0,0 +1,128 @@ +# Cloudflare R2 SQL Skill Reference + +Expert guidance for Cloudflare R2 SQL - serverless distributed query engine for Apache Iceberg tables. + +## Reading Order + +**New to R2 SQL?** Start here: +1. Read "What is R2 SQL?" and "When to Use" below +2. [configuration.md](configuration.md) - Enable catalog, create tokens +3. [patterns.md](patterns.md) - Wrangler CLI and integration examples +4. [api.md](api.md) - SQL syntax and query reference +5. [gotchas.md](gotchas.md) - Limitations and troubleshooting + +**Quick reference?** Jump to: +- [Run a query via Wrangler](patterns.md#wrangler-cli-query) +- [SQL syntax reference](api.md#sql-syntax) +- [ORDER BY limitations](gotchas.md#order-by-limitations) + +## What is R2 SQL? + +R2 SQL is Cloudflare's **serverless distributed analytics query engine** for querying Apache Iceberg tables in R2 Data Catalog. Features: + +- **Serverless** - No clusters to manage, no infrastructure +- **Distributed** - Leverages Cloudflare's global network for parallel execution +- **SQL interface** - Familiar SQL syntax for analytics queries +- **Zero egress fees** - Query from any cloud/region without data transfer costs +- **Open beta** - Free during beta (standard R2 storage costs apply) + +### What is Apache Iceberg? + +Open table format for large-scale analytics datasets in object storage: +- **ACID transactions** - Safe concurrent reads/writes +- **Metadata optimization** - Fast queries without full table scans +- **Schema evolution** - Add/rename/drop columns without rewrites +- **Partitioning** - Organize data for efficient pruning + +## When to Use + +**Use R2 SQL for:** +- **Log analytics** - Query application/system logs with WHERE filters and aggregations +- **BI dashboards** - Generate reports from large analytical datasets +- **Fraud detection** - Analyze transaction patterns with GROUP BY/HAVING +- **Multi-cloud analytics** - Query data from any cloud without egress fees +- **Ad-hoc exploration** - Run SQL queries on Iceberg tables via Wrangler CLI + +**Don't use R2 SQL for:** +- **Workers/Pages runtime** - R2 SQL has no Workers binding, use HTTP API from external systems +- **Real-time queries (<100ms)** - Optimized for analytical batch queries, not OLTP +- **Complex joins/CTEs** - Limited SQL feature set (no JOINs, subqueries, CTEs currently) +- **Small datasets (<1GB)** - Setup overhead not justified + +## Decision Tree: Need to Query R2 Data? + +``` +Do you need to query structured data in R2? +├─ YES, data is in Iceberg tables +│ ├─ Need SQL interface? → Use R2 SQL (this reference) +│ ├─ Need Python API? → See r2-data-catalog reference (PyIceberg) +│ └─ Need other engine? → See r2-data-catalog reference (Spark, Trino, etc.) +│ +├─ YES, but not in Iceberg format +│ ├─ Streaming data? → Use Pipelines to write to Data Catalog, then R2 SQL +│ └─ Static files? → Use PyIceberg to create Iceberg tables, then R2 SQL +│ +└─ NO, just need object storage → Use R2 reference (not R2 SQL) +``` + +## Architecture Overview + +**Query Planner:** +- Top-down metadata investigation with multi-layer pruning +- Partition-level, column-level, and row-group pruning +- Streaming pipeline - execution starts before planning completes +- Early termination with LIMIT - stops when result complete + +**Query Execution:** +- Coordinator distributes work to workers across Cloudflare network +- Workers run Apache DataFusion for parallel query execution +- Parquet column pruning - reads only required columns +- Ranged reads from R2 for efficiency + +**Aggregation Strategies:** +- Scatter-gather - simple aggregations (SUM, COUNT, AVG) +- Shuffling - ORDER BY/HAVING on aggregates via hash partitioning + +## Quick Start + +```bash +# 1. Enable R2 Data Catalog on bucket +npx wrangler r2 bucket catalog enable my-bucket + +# 2. Create API token (Admin Read & Write) +# Dashboard: R2 → Manage API tokens → Create API token + +# 3. Set environment variable +export WRANGLER_R2_SQL_AUTH_TOKEN= + +# 4. Run query +npx wrangler r2 sql query "my-bucket" "SELECT * FROM default.my_table LIMIT 10" +``` + +## Important Limitations + +**CRITICAL: No Workers Binding** +- R2 SQL cannot be called directly from Workers/Pages code +- For programmatic access, use HTTP API from external systems +- Or query via PyIceberg, Spark, etc. (see r2-data-catalog reference) + +**SQL Feature Set:** +- No JOINs, CTEs, subqueries, window functions +- ORDER BY supports aggregation columns (not just partition keys) +- LIMIT max 10,000 (default 500) +- See [gotchas.md](gotchas.md) for complete limitations + +## In This Reference + +- **[configuration.md](configuration.md)** - Enable catalog, create API tokens +- **[api.md](api.md)** - SQL syntax, functions, operators, data types +- **[patterns.md](patterns.md)** - Wrangler CLI, HTTP API, Pipelines, PyIceberg +- **[gotchas.md](gotchas.md)** - Limitations, troubleshooting, performance tips + +## See Also + +- [r2-data-catalog](../r2-data-catalog/) - PyIceberg, REST API, external engines +- [pipelines](../pipelines/) - Streaming ingestion to Iceberg tables +- [r2](../r2/) - R2 object storage fundamentals +- [Cloudflare R2 SQL Docs](https://developers.cloudflare.com/r2-sql/) +- [R2 SQL Deep Dive Blog](https://blog.cloudflare.com/r2-sql-deep-dive/) diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-sql/api.md b/plugins/cloudflare/skills/cloudflare/references/r2-sql/api.md new file mode 100644 index 0000000..7e67c8f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-sql/api.md @@ -0,0 +1,158 @@ +# R2 SQL API Reference + +SQL syntax, functions, operators, and data types for R2 SQL queries. + +## SQL Syntax + +```sql +SELECT column_list | aggregation_function +FROM [namespace.]table_name +WHERE conditions +[GROUP BY column_list] +[HAVING conditions] +[ORDER BY column | aggregation_function [DESC | ASC]] +[LIMIT number] +``` + +## Schema Discovery + +```sql +SHOW DATABASES; -- List namespaces +SHOW NAMESPACES; -- Alias for SHOW DATABASES +SHOW SCHEMAS; -- Alias for SHOW DATABASES +SHOW TABLES IN namespace; -- List tables in namespace +DESCRIBE namespace.table; -- Show table schema, partition keys +``` + +## SELECT Clause + +```sql +-- All columns +SELECT * FROM logs.http_requests; + +-- Specific columns +SELECT user_id, timestamp, status FROM logs.http_requests; +``` + +**Limitations:** No column aliases, expressions, or nested column access + +## WHERE Clause + +### Operators + +| Operator | Example | +|----------|---------| +| `=`, `!=`, `<`, `<=`, `>`, `>=` | `status = 200` | +| `LIKE` | `user_agent LIKE '%Chrome%'` | +| `BETWEEN` | `timestamp BETWEEN '2025-01-01T00:00:00Z' AND '2025-01-31T23:59:59Z'` | +| `IS NULL`, `IS NOT NULL` | `email IS NOT NULL` | +| `AND`, `OR` | `status = 200 AND method = 'GET'` | + +Use parentheses for precedence: `(status = 404 OR status = 500) AND method = 'POST'` + +## Aggregation Functions + +| Function | Description | +|----------|-------------| +| `COUNT(*)` | Count all rows | +| `COUNT(column)` | Count non-null values | +| `COUNT(DISTINCT column)` | Count unique values | +| `SUM(column)`, `AVG(column)` | Numeric aggregations | +| `MIN(column)`, `MAX(column)` | Min/max values | + +```sql +-- Multiple aggregations with GROUP BY +SELECT region, COUNT(*), SUM(amount), AVG(amount) +FROM sales.transactions +WHERE sale_date >= '2024-01-01' +GROUP BY region; +``` + +## HAVING Clause + +Filter aggregated results (after GROUP BY): + +```sql +SELECT category, SUM(amount) +FROM sales.transactions +GROUP BY category +HAVING SUM(amount) > 10000; +``` + +## ORDER BY Clause + +Sort results by: +- **Partition key columns** - Always supported +- **Aggregation functions** - Supported via shuffle strategy + +```sql +-- Order by partition key +SELECT * FROM logs.requests ORDER BY timestamp DESC LIMIT 100; + +-- Order by aggregation (repeat function, aliases not supported) +SELECT region, SUM(amount) +FROM sales.transactions +GROUP BY region +ORDER BY SUM(amount) DESC; +``` + +**Limitations:** Cannot order by non-partition columns. See [gotchas.md](gotchas.md#order-by-limitations) + +## LIMIT Clause + +```sql +SELECT * FROM logs.requests LIMIT 100; +``` + +| Setting | Value | +|---------|-------| +| Min | 1 | +| Max | 10,000 | +| Default | 500 | + +**Always use LIMIT** to enable early termination optimization. + +## Data Types + +| Type | SQL Literal | Example | +|------|-------------|---------| +| `integer` | Unquoted number | `42`, `-10` | +| `float` | Decimal number | `3.14`, `-0.5` | +| `string` | Single quotes | `'hello'`, `'GET'` | +| `boolean` | Keyword | `true`, `false` | +| `timestamp` | RFC3339 string | `'2025-01-01T00:00:00Z'` | +| `date` | ISO 8601 date | `'2025-01-01'` | + +### Type Safety + +- Quote strings with single quotes: `'value'` +- Timestamps must be RFC3339: `'2025-01-01T00:00:00Z'` (include timezone) +- Dates must be ISO 8601: `'2025-01-01'` (YYYY-MM-DD) +- No implicit conversions + +```sql +-- ✅ Correct +WHERE status = 200 AND method = 'GET' AND timestamp > '2025-01-01T00:00:00Z' + +-- ❌ Wrong +WHERE status = '200' -- string instead of integer +WHERE timestamp > '2025-01-01' -- missing time/timezone +WHERE method = GET -- unquoted string +``` + +## Query Result Format + +JSON array of objects: + +```json +[ + {"user_id": "user_123", "timestamp": "2025-01-15T10:30:00Z", "status": 200}, + {"user_id": "user_456", "timestamp": "2025-01-15T10:31:00Z", "status": 404} +] +``` + +## See Also + +- [patterns.md](patterns.md) - Query examples and use cases +- [gotchas.md](gotchas.md) - SQL limitations and error handling +- [configuration.md](configuration.md) - Setup and authentication diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-sql/configuration.md b/plugins/cloudflare/skills/cloudflare/references/r2-sql/configuration.md new file mode 100644 index 0000000..3c5cfb2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-sql/configuration.md @@ -0,0 +1,147 @@ +# R2 SQL Configuration + +Setup and configuration for R2 SQL queries. + +## Prerequisites + +- R2 bucket with Data Catalog enabled +- API token with R2 permissions +- Wrangler CLI installed (for CLI queries) + +## Enable R2 Data Catalog + +R2 SQL queries Apache Iceberg tables in R2 Data Catalog. Must enable catalog on bucket first. + +### Via Wrangler CLI + +```bash +npx wrangler r2 bucket catalog enable +``` + +Output includes: +- **Warehouse name** - Typically same as bucket name +- **Catalog URI** - REST endpoint for catalog operations + +Example output: +``` +Catalog enabled successfully +Warehouse: my-bucket +Catalog URI: https://abc123.r2.cloudflarestorage.com/iceberg/my-bucket +``` + +### Via Dashboard + +1. Navigate to **R2 Object Storage** → Select your bucket +2. Click **Settings** tab +3. Scroll to **R2 Data Catalog** section +4. Click **Enable** +5. Note the **Catalog URI** and **Warehouse** name + +**Important:** Enabling catalog creates metadata directories in bucket but does not modify existing objects. + +## Create API Token + +R2 SQL requires API token with R2 permissions. + +### Required Permission + +**R2 Admin Read & Write** (includes R2 SQL Read permission) + +### Via Dashboard + +1. Navigate to **R2 Object Storage** +2. Click **Manage API tokens** (top right) +3. Click **Create API token** +4. Select **Admin Read & Write** permission +5. Click **Create API Token** +6. **Copy token value** - shown only once + +### Permission Scope + +| Permission | Grants Access To | +|------------|------------------| +| R2 Admin Read & Write | R2 storage operations + R2 SQL queries + Data Catalog operations | +| R2 SQL Read | SQL queries only (no storage writes) | + +**Note:** R2 SQL Read permission not yet available via Dashboard - use Admin Read & Write. + +## Configure Environment + +### Wrangler CLI + +Set environment variable for Wrangler to use: + +```bash +export WRANGLER_R2_SQL_AUTH_TOKEN= +``` + +Or create `.env` file in project directory: + +``` +WRANGLER_R2_SQL_AUTH_TOKEN= +``` + +Wrangler automatically loads `.env` file when running commands. + +### HTTP API + +For programmatic access (non-Wrangler), pass token in Authorization header: + +```bash +curl -X POST https://api.cloudflare.com/client/v4/accounts/{account_id}/r2/sql/query \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "warehouse": "my-bucket", + "query": "SELECT * FROM default.my_table LIMIT 10" + }' +``` + +**Note:** HTTP API endpoint URL may vary - see [patterns.md](patterns.md#http-api-query) for current endpoint. + +## Verify Setup + +Test configuration by querying system tables: + +```bash +# List namespaces +npx wrangler r2 sql query "my-bucket" "SHOW DATABASES" + +# List tables in namespace +npx wrangler r2 sql query "my-bucket" "SHOW TABLES IN default" +``` + +If successful, returns JSON array of results. + +## Troubleshooting + +### "Token authentication failed" + +**Cause:** Invalid or missing token + +**Solution:** +- Verify `WRANGLER_R2_SQL_AUTH_TOKEN` environment variable set +- Check token has Admin Read & Write permission +- Create new token if expired + +### "Catalog not enabled on bucket" + +**Cause:** Data Catalog not enabled + +**Solution:** +- Run `npx wrangler r2 bucket catalog enable ` +- Or enable via Dashboard (R2 → bucket → Settings → R2 Data Catalog) + +### "Permission denied" + +**Cause:** Token lacks required permissions + +**Solution:** +- Verify token has **Admin Read & Write** permission +- Create new token with correct permissions + +## See Also + +- [r2-data-catalog/configuration.md](../r2-data-catalog/configuration.md) - Detailed token setup and PyIceberg connection +- [patterns.md](patterns.md) - Query examples using configuration +- [gotchas.md](gotchas.md) - Common configuration errors diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-sql/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/r2-sql/gotchas.md new file mode 100644 index 0000000..d16de94 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-sql/gotchas.md @@ -0,0 +1,212 @@ +# R2 SQL Gotchas + +Limitations, troubleshooting, and common pitfalls for R2 SQL. + +## Critical Limitations + +### No Workers Binding + +**Cannot call R2 SQL from Workers/Pages code** - no binding exists. + +```typescript +// ❌ This doesn't exist +export default { + async fetch(request, env) { + const result = await env.R2_SQL.query("SELECT * FROM table"); // Not possible + return Response.json(result); + } +}; +``` + +**Solutions:** +- HTTP API from external systems (not Workers) +- PyIceberg/Spark via r2-data-catalog REST API +- For Workers, use D1 or external databases + +### ORDER BY Limitations + +Can only order by: +1. **Partition key columns** - Always supported +2. **Aggregation functions** - Supported via shuffle strategy + +**Cannot order by** regular non-partition columns. + +```sql +-- ✅ Valid: ORDER BY partition key +SELECT * FROM logs.requests ORDER BY timestamp DESC LIMIT 100; + +-- ✅ Valid: ORDER BY aggregation +SELECT region, SUM(amount) FROM sales.transactions +GROUP BY region ORDER BY SUM(amount) DESC; + +-- ❌ Invalid: ORDER BY non-partition column +SELECT * FROM logs.requests ORDER BY user_id; + +-- ❌ Invalid: ORDER BY alias (must repeat function) +SELECT region, SUM(amount) as total FROM sales.transactions +GROUP BY region ORDER BY total; -- Use ORDER BY SUM(amount) +``` + +Check partition spec: `DESCRIBE namespace.table_name` + +## SQL Feature Limitations + +| Feature | Supported | Notes | +|---------|-----------|-------| +| SELECT, WHERE, GROUP BY, HAVING | ✅ | Standard support | +| COUNT, SUM, AVG, MIN, MAX | ✅ | Standard aggregations | +| ORDER BY partition/aggregation | ✅ | See above | +| LIMIT | ✅ | Max 10,000 | +| Column aliases | ❌ | No AS alias | +| Expressions in SELECT | ❌ | No col1 + col2 | +| ORDER BY non-partition | ❌ | Fails at runtime | +| JOINs, subqueries, CTEs | ❌ | Denormalize at write time | +| Window functions, UNION | ❌ | Use external engines | +| INSERT/UPDATE/DELETE | ❌ | Use PyIceberg/Pipelines | +| Nested columns, arrays, JSON | ❌ | Flatten at write time | + +**Workarounds:** +- No JOINs: Denormalize data or use Spark/PyIceberg +- No subqueries: Split into multiple queries +- No aliases: Accept generated names, transform in app + +## Common Errors + +### "Column not found" +**Cause:** Typo, column doesn't exist, or case mismatch +**Solution:** `DESCRIBE namespace.table_name` to check schema + +### "Type mismatch" +```sql +-- ❌ Wrong types +WHERE status = '200' -- string instead of integer +WHERE timestamp > '2025-01-01' -- missing time/timezone + +-- ✅ Correct types +WHERE status = 200 +WHERE timestamp > '2025-01-01T00:00:00Z' +``` + +### "ORDER BY column not in partition key" +**Cause:** Ordering by non-partition column +**Solution:** Use partition key, aggregation, or remove ORDER BY. Check: `DESCRIBE table` + +### "Token authentication failed" +```bash +# Check/set token +echo $WRANGLER_R2_SQL_AUTH_TOKEN +export WRANGLER_R2_SQL_AUTH_TOKEN= + +# Or .env file +echo "WRANGLER_R2_SQL_AUTH_TOKEN=" > .env +``` + +### "Table not found" +```sql +-- Verify catalog and tables +SHOW DATABASES; +SHOW TABLES IN namespace_name; +``` + +Enable catalog: `npx wrangler r2 bucket catalog enable ` + +### "LIMIT exceeds maximum" +Max LIMIT is 10,000. For pagination, use WHERE filters with partition keys. + +### "No data returned" (unexpected) +**Debug steps:** +1. `SELECT COUNT(*) FROM table` - verify data exists +2. Remove WHERE filters incrementally +3. `SELECT * FROM table LIMIT 10` - inspect actual data/types + +## Performance Issues + +### Slow Queries + +**Causes:** Too many partitions, large LIMIT, no filters, small files + +```sql +-- ❌ Slow: No filters +SELECT * FROM logs.requests LIMIT 10000; + +-- ✅ Fast: Filter on partition key +SELECT * FROM logs.requests +WHERE timestamp >= '2025-01-15T00:00:00Z' AND timestamp < '2025-01-16T00:00:00Z' +LIMIT 1000; + +-- ✅ Faster: Multiple filters +SELECT * FROM logs.requests +WHERE timestamp >= '2025-01-15T00:00:00Z' AND status = 404 AND method = 'GET' +LIMIT 1000; +``` + +**File optimization:** +- Target Parquet size: 100-500MB compressed +- Pipelines roll interval: 300+ sec (prod), 10 sec (dev) +- Run compaction to merge small files + +### Query Timeout + +**Solution:** Add restrictive WHERE filters, reduce time range, query smaller intervals + +```sql +-- ❌ Times out: Year-long aggregation +SELECT status, COUNT(*) FROM logs.requests +WHERE timestamp >= '2024-01-01T00:00:00Z' GROUP BY status; + +-- ✅ Faster: Month-long aggregation +SELECT status, COUNT(*) FROM logs.requests +WHERE timestamp >= '2025-01-01T00:00:00Z' AND timestamp < '2025-02-01T00:00:00Z' +GROUP BY status; +``` + +## Best Practices + +### Partitioning +- **Time-series:** Partition by day/hour on timestamp +- **Avoid:** High-cardinality keys (user_id), >10,000 partitions + +```python +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform + +PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name="day")) +``` + +### Query Writing +- **Always use LIMIT** for early termination +- **Filter on partition keys first** for pruning +- **Combine filters with AND** for more pruning + +```sql +-- Good +WHERE timestamp >= '2025-01-15T00:00:00Z' AND status = 404 AND method = 'GET' LIMIT 100 +``` + +### Type Safety +- Quote strings: `'GET'` not `GET` +- RFC3339 timestamps: `'2025-01-01T00:00:00Z'` not `'2025-01-01'` +- ISO dates: `'2025-01-15'` not `'01/15/2025'` + +### Data Organization +- **Pipelines:** Dev `roll_file_time: 10`, Prod `roll_file_time: 300+` +- **Compression:** Use `zstd` +- **Maintenance:** Compaction for small files, expire old snapshots + +## Debugging Checklist + +1. `npx wrangler r2 bucket catalog enable ` - Verify catalog +2. `echo $WRANGLER_R2_SQL_AUTH_TOKEN` - Check token +3. `SHOW DATABASES` - List namespaces +4. `SHOW TABLES IN namespace` - List tables +5. `DESCRIBE namespace.table` - Check schema +6. `SELECT COUNT(*) FROM namespace.table` - Verify data +7. `SELECT * FROM namespace.table LIMIT 10` - Test simple query +8. Add filters incrementally + +## See Also + +- [api.md](api.md) - SQL syntax +- [patterns.md](patterns.md) - Query optimization +- [configuration.md](configuration.md) - Setup +- [Cloudflare R2 SQL Docs](https://developers.cloudflare.com/r2-sql/) diff --git a/plugins/cloudflare/skills/cloudflare/references/r2-sql/patterns.md b/plugins/cloudflare/skills/cloudflare/references/r2-sql/patterns.md new file mode 100644 index 0000000..53de7d3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2-sql/patterns.md @@ -0,0 +1,222 @@ +# R2 SQL Patterns + +Common patterns, use cases, and integration examples for R2 SQL. + +## Wrangler CLI Query + +```bash +# Basic query +npx wrangler r2 sql query "my-bucket" "SELECT * FROM default.logs LIMIT 10" + +# Multi-line query +npx wrangler r2 sql query "my-bucket" " + SELECT status, COUNT(*), AVG(response_time) + FROM logs.http_requests + WHERE timestamp >= '2025-01-01T00:00:00Z' + GROUP BY status + ORDER BY COUNT(*) DESC + LIMIT 100 +" + +# Use environment variable +export R2_SQL_WAREHOUSE="my-bucket" +npx wrangler r2 sql query "$R2_SQL_WAREHOUSE" "SELECT * FROM default.logs" +``` + +## HTTP API Query + +For programmatic access from external systems (not Workers - see gotchas.md). + +```bash +curl -X POST https://api.cloudflare.com/client/v4/accounts/{account_id}/r2/sql/query \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "warehouse": "my-bucket", + "query": "SELECT * FROM default.my_table WHERE status = 200 LIMIT 100" + }' +``` + +Response: +```json +{ + "success": true, + "result": [{"user_id": "user_123", "timestamp": "2025-01-15T10:30:00Z", "status": 200}], + "errors": [] +} +``` + +## Pipelines Integration + +Stream data to Iceberg tables via Pipelines, then query with R2 SQL. + +```bash +# Setup pipeline (select Data Catalog Table destination) +npx wrangler pipelines setup + +# Key settings: +# - Destination: Data Catalog Table +# - Compression: zstd (recommended) +# - Roll file time: 300+ sec (production), 10 sec (dev) + +# Send data to pipeline +curl -X POST https://{stream-id}.ingest.cloudflare.com \ + -H "Content-Type: application/json" \ + -d '[{"user_id": "user_123", "event_type": "purchase", "timestamp": "2025-01-15T10:30:00Z", "amount": 29.99}]' + +# Query ingested data (wait for roll interval) +npx wrangler r2 sql query "my-bucket" " + SELECT event_type, COUNT(*), SUM(amount) + FROM default.events + WHERE timestamp >= '2025-01-15T00:00:00Z' + GROUP BY event_type +" +``` + +See [pipelines/patterns.md](../pipelines/patterns.md) for detailed setup. + +## PyIceberg Integration + +Create and populate Iceberg tables with PyIceberg, then query with R2 SQL. + +```python +from pyiceberg.catalog.rest import RestCatalog +import pyarrow as pa +import pandas as pd + +# Setup catalog +catalog = RestCatalog( + name="my_catalog", + warehouse="my-bucket", + uri="https://.r2.cloudflarestorage.com/iceberg/my-bucket", + token="", +) +catalog.create_namespace_if_not_exists("analytics") + +# Create table +schema = pa.schema([ + pa.field("user_id", pa.string(), nullable=False), + pa.field("event_time", pa.timestamp("us", tz="UTC"), nullable=False), + pa.field("page_views", pa.int64(), nullable=False), +]) +table = catalog.create_table(("analytics", "user_metrics"), schema=schema) + +# Append data +df = pd.DataFrame({ + "user_id": ["user_1", "user_2"], + "event_time": pd.to_datetime(["2025-01-15 10:00:00", "2025-01-15 11:00:00"], utc=True), + "page_views": [10, 25], +}) +table.append(pa.Table.from_pandas(df, schema=schema)) +``` + +Query with R2 SQL: +```bash +npx wrangler r2 sql query "my-bucket" " + SELECT user_id, SUM(page_views) + FROM analytics.user_metrics + WHERE event_time >= '2025-01-15T00:00:00Z' + GROUP BY user_id +" +``` + +See [r2-data-catalog/patterns.md](../r2-data-catalog/patterns.md) for advanced PyIceberg patterns. + +## Use Cases + +### Log Analytics +```sql +-- Error rate by endpoint +SELECT path, COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) as errors +FROM logs.http_requests +WHERE timestamp BETWEEN '2025-01-01T00:00:00Z' AND '2025-01-31T23:59:59Z' +GROUP BY path ORDER BY errors DESC LIMIT 20; + +-- Response time stats +SELECT method, MIN(response_time_ms), AVG(response_time_ms), MAX(response_time_ms) +FROM logs.http_requests WHERE timestamp >= '2025-01-15T00:00:00Z' GROUP BY method; + +-- Traffic by status +SELECT status, COUNT(*) FROM logs.http_requests +WHERE timestamp >= '2025-01-15T00:00:00Z' AND method = 'GET' +GROUP BY status ORDER BY COUNT(*) DESC; +``` + +### Fraud Detection +```sql +-- High-value transactions +SELECT location, COUNT(*), SUM(amount), AVG(amount) +FROM fraud.transactions WHERE transaction_timestamp >= '2025-01-01T00:00:00Z' AND amount > 1000.0 +GROUP BY location ORDER BY SUM(amount) DESC LIMIT 20; + +-- Flagged transactions +SELECT merchant_category, COUNT(*), AVG(amount) FROM fraud.transactions +WHERE is_fraud_flag = true AND transaction_timestamp >= '2025-01-01T00:00:00Z' +GROUP BY merchant_category HAVING COUNT(*) > 10 ORDER BY COUNT(*) DESC; +``` + +### Business Intelligence +```sql +-- Sales by department +SELECT department, SUM(revenue), AVG(revenue), COUNT(*) FROM sales.transactions +WHERE sale_date >= '2024-01-01' GROUP BY department ORDER BY SUM(revenue) DESC LIMIT 10; + +-- Product performance +SELECT category, COUNT(DISTINCT product_id), SUM(units_sold), SUM(revenue) +FROM sales.product_sales WHERE sale_date BETWEEN '2024-10-01' AND '2024-12-31' +GROUP BY category ORDER BY SUM(revenue) DESC; +``` + +## Connecting External Engines + +R2 Data Catalog exposes Iceberg REST API. Connect Spark, Snowflake, Trino, DuckDB, etc. + +```scala +// Apache Spark example +val spark = SparkSession.builder() + .config("spark.sql.catalog.my_catalog", "org.apache.iceberg.spark.SparkCatalog") + .config("spark.sql.catalog.my_catalog.catalog-impl", "org.apache.iceberg.rest.RESTCatalog") + .config("spark.sql.catalog.my_catalog.uri", "https://.r2.cloudflarestorage.com/iceberg/my-bucket") + .config("spark.sql.catalog.my_catalog.token", "") + .getOrCreate() + +spark.sql("SELECT * FROM my_catalog.default.my_table LIMIT 10").show() +``` + +See [r2-data-catalog/patterns.md](../r2-data-catalog/patterns.md) for more engines. + +## Performance Optimization + +### Partitioning +- **Time-series:** day/hour on timestamp +- **Geographic:** region/country +- **Avoid:** High-cardinality keys (user_id) + +```python +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform + +PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name="day")) +``` + +### Query Optimization +- **Always use LIMIT** for early termination +- **Filter on partition keys first** +- **Multiple filters** for better pruning + +```sql +-- Better: Multiple filters on partition key +SELECT * FROM logs.requests +WHERE timestamp >= '2025-01-15T00:00:00Z' AND status = 404 AND method = 'GET' LIMIT 100; +``` + +### File Organization +- **Pipelines roll:** Dev 10-30s, Prod 300+s +- **Target Parquet:** 100-500MB compressed + +## See Also + +- [api.md](api.md) - SQL syntax reference +- [gotchas.md](gotchas.md) - Limitations and troubleshooting +- [r2-data-catalog/patterns.md](../r2-data-catalog/patterns.md) - PyIceberg advanced patterns +- [pipelines/patterns.md](../pipelines/patterns.md) - Streaming ingestion patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/r2/README.md b/plugins/cloudflare/skills/cloudflare/references/r2/README.md new file mode 100644 index 0000000..af3d7d0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2/README.md @@ -0,0 +1,95 @@ +# Cloudflare R2 Object Storage + +S3-compatible object storage with zero egress fees, optimized for large file storage and delivery. + +## Overview + +R2 provides: +- S3-compatible API (Workers API + S3 REST) +- Zero egress fees globally +- Strong consistency for writes/deletes +- Storage classes (Standard/Infrequent Access) +- SSE-C encryption support + +**Use cases:** Media storage, backups, static assets, user uploads, data lakes + +## Quick Start + +```bash +wrangler r2 bucket create my-bucket --location=enam +wrangler r2 object put my-bucket/file.txt --file=./local.txt +``` + +```typescript +// Upload +await env.MY_BUCKET.put(key, data, { + httpMetadata: { contentType: 'image/jpeg' } +}); + +// Download +const object = await env.MY_BUCKET.get(key); +if (object) return new Response(object.body); +``` + +## Core Operations + +| Method | Purpose | Returns | +|--------|---------|---------| +| `put(key, value, options?)` | Upload object | `R2Object \| null` | +| `get(key, options?)` | Download object | `R2ObjectBody \| R2Object \| null` | +| `head(key)` | Get metadata only | `R2Object \| null` | +| `delete(keys)` | Delete object(s) | `Promise` | +| `list(options?)` | List objects | `R2Objects` | + +## Storage Classes + +- **Standard**: Frequent access, low latency reads +- **InfrequentAccess**: 30-day minimum storage, retrieval fees, lower storage cost + +## Event Notifications + +R2 integrates with Cloudflare Queues for reactive workflows: + +```typescript +// wrangler.jsonc +{ + "event_notifications": [{ + "queue": "r2-notifications", + "actions": ["PutObject", "DeleteObject"] + }] +} + +// Consumer +async queue(batch: MessageBatch, env: Env) { + for (const message of batch.messages) { + const event = message.body; // { action, bucket, object, timestamps } + if (event.action === 'PutObject') { + // Process upload: thumbnail generation, virus scan, etc. + } + } +} +``` + +## Reading Order + +**First-time users:** README → configuration.md → api.md → patterns.md +**Specific tasks:** +- Setup: configuration.md +- Client uploads: patterns.md (presigned URLs) +- Public static site: patterns.md (public access + custom domain) +- Processing uploads: README (event notifications) + queues reference +- Debugging: gotchas.md + +## In This Reference + +- [configuration.md](./configuration.md) - Bindings, S3 SDK, CORS, lifecycles, token scopes +- [api.md](./api.md) - Workers API, multipart, conditional requests, presigned URLs +- [patterns.md](./patterns.md) - Streaming, caching, client uploads, public buckets +- [gotchas.md](./gotchas.md) - List truncation, etag format, stream length, S3 SDK region + +## See Also + +- [workers](../workers/) - Worker runtime and fetch handlers +- [kv](../kv/) - Metadata storage for R2 objects +- [d1](../d1/) - Store R2 URLs in relational database +- [queues](../queues/) - Process R2 uploads asynchronously diff --git a/plugins/cloudflare/skills/cloudflare/references/r2/api.md b/plugins/cloudflare/skills/cloudflare/references/r2/api.md new file mode 100644 index 0000000..9a8cd28 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2/api.md @@ -0,0 +1,200 @@ +# R2 API Reference + +## PUT (Upload) + +```typescript +// Basic +await env.MY_BUCKET.put(key, value); + +// With metadata +await env.MY_BUCKET.put(key, value, { + httpMetadata: { + contentType: 'image/jpeg', + contentDisposition: 'attachment; filename="photo.jpg"', + cacheControl: 'max-age=3600' + }, + customMetadata: { userId: '123', version: '2' }, + storageClass: 'Standard', // or 'InfrequentAccess' + sha256: arrayBufferOrHex, // Integrity check + ssecKey: arrayBuffer32bytes // SSE-C encryption +}); + +// Value types: ReadableStream | ArrayBuffer | string | Blob +``` + +## GET (Download) + +```typescript +const object = await env.MY_BUCKET.get(key); +if (!object) return new Response('Not found', { status: 404 }); + +// Body: arrayBuffer(), text(), json(), blob(), body (ReadableStream) + +// Ranged reads +const object = await env.MY_BUCKET.get(key, { range: { offset: 0, length: 1024 } }); + +// Conditional GET +const object = await env.MY_BUCKET.get(key, { onlyIf: { etagMatches: '"abc123"' } }); +``` + +## HEAD (Metadata Only) + +```typescript +const object = await env.MY_BUCKET.head(key); // Returns R2Object without body +``` + +## DELETE + +```typescript +await env.MY_BUCKET.delete(key); +await env.MY_BUCKET.delete([key1, key2, key3]); // Batch (max 1000) +``` +## LIST + +```typescript +const listed = await env.MY_BUCKET.list({ + limit: 1000, + prefix: 'photos/', + cursor: cursorFromPrevious, + delimiter: '/', + include: ['httpMetadata', 'customMetadata'] +}); + +// Pagination (always use truncated flag) +while (listed.truncated) { + const next = await env.MY_BUCKET.list({ cursor: listed.cursor }); + listed.objects.push(...next.objects); + listed.truncated = next.truncated; + listed.cursor = next.cursor; +} +``` + +## Multipart Uploads + +```typescript +const multipart = await env.MY_BUCKET.createMultipartUpload(key, { + httpMetadata: { contentType: 'video/mp4' } +}); + +const uploadedParts: R2UploadedPart[] = []; +for (let i = 0; i < partCount; i++) { + const part = await multipart.uploadPart(i + 1, partData); + uploadedParts.push(part); +} + +const object = await multipart.complete(uploadedParts); +// OR: await multipart.abort(); + +// Resume +const multipart = env.MY_BUCKET.resumeMultipartUpload(key, uploadId); +``` + +## Presigned URLs (S3 SDK) + +```typescript +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; + +const s3 = new S3Client({ + region: 'auto', + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + credentials: { accessKeyId: env.R2_ACCESS_KEY_ID, secretAccessKey: env.R2_SECRET_ACCESS_KEY } +}); + +const uploadUrl = await getSignedUrl(s3, new PutObjectCommand({ Bucket: 'my-bucket', Key: key }), { expiresIn: 3600 }); +return Response.json({ uploadUrl }); +``` + +## TypeScript Interfaces + +```typescript +interface R2Bucket { + head(key: string): Promise; + get(key: string, options?: R2GetOptions): Promise; + put(key: string, value: ReadableStream | ArrayBuffer | string | Blob, options?: R2PutOptions): Promise; + delete(keys: string | string[]): Promise; + list(options?: R2ListOptions): Promise; + createMultipartUpload(key: string, options?: R2MultipartOptions): Promise; + resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload; +} + +interface R2Object { + key: string; version: string; size: number; + etag: string; httpEtag: string; // httpEtag is quoted, use for headers + uploaded: Date; httpMetadata?: R2HTTPMetadata; + customMetadata?: Record; + storageClass: 'Standard' | 'InfrequentAccess'; + checksums: R2Checksums; + writeHttpMetadata(headers: Headers): void; +} + +interface R2ObjectBody extends R2Object { + body: ReadableStream; bodyUsed: boolean; + arrayBuffer(): Promise; text(): Promise; + json(): Promise; blob(): Promise; +} + +interface R2HTTPMetadata { + contentType?: string; contentDisposition?: string; + contentEncoding?: string; contentLanguage?: string; + cacheControl?: string; cacheExpiry?: Date; +} + +interface R2PutOptions { + httpMetadata?: R2HTTPMetadata | Headers; + customMetadata?: Record; + sha256?: ArrayBuffer | string; // Only ONE checksum allowed + storageClass?: 'Standard' | 'InfrequentAccess'; + ssecKey?: ArrayBuffer; +} + +interface R2GetOptions { + onlyIf?: R2Conditional | Headers; + range?: R2Range | Headers; + ssecKey?: ArrayBuffer; +} + +interface R2ListOptions { + limit?: number; prefix?: string; cursor?: string; delimiter?: string; + startAfter?: string; include?: ('httpMetadata' | 'customMetadata')[]; +} + +interface R2Objects { + objects: R2Object[]; truncated: boolean; + cursor?: string; delimitedPrefixes: string[]; +} + +interface R2Conditional { + etagMatches?: string; etagDoesNotMatch?: string; + uploadedBefore?: Date; uploadedAfter?: Date; +} + +interface R2Range { offset?: number; length?: number; suffix?: number; } + +interface R2Checksums { + md5?: ArrayBuffer; sha1?: ArrayBuffer; sha256?: ArrayBuffer; + sha384?: ArrayBuffer; sha512?: ArrayBuffer; +} + +interface R2MultipartUpload { + key: string; + uploadId: string; + uploadPart(partNumber: number, value: ReadableStream | ArrayBuffer | string | Blob): Promise; + abort(): Promise; + complete(uploadedParts: R2UploadedPart[]): Promise; +} + +interface R2UploadedPart { + partNumber: number; + etag: string; +} +``` + +## CLI Operations + +```bash +wrangler r2 object put my-bucket/file.txt --file=./local.txt +wrangler r2 object get my-bucket/file.txt --file=./download.txt +wrangler r2 object delete my-bucket/file.txt +wrangler r2 object list my-bucket --prefix=photos/ +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/r2/configuration.md b/plugins/cloudflare/skills/cloudflare/references/r2/configuration.md new file mode 100644 index 0000000..f306acd --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2/configuration.md @@ -0,0 +1,165 @@ +# R2 Configuration + +## Workers Binding + +**wrangler.jsonc:** +```jsonc +{ + "r2_buckets": [ + { + "binding": "MY_BUCKET", + "bucket_name": "my-bucket-name" + } + ] +} +``` + +## TypeScript Types + +```typescript +interface Env { MY_BUCKET: R2Bucket; } + +export default { + async fetch(request: Request, env: Env): Promise { + const object = await env.MY_BUCKET.get('file.txt'); + return new Response(object?.body); + } +} +``` + +## S3 SDK Setup + +```typescript +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; + +const s3 = new S3Client({ + region: 'auto', + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + credentials: { + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY + } +}); + +await s3.send(new PutObjectCommand({ + Bucket: 'my-bucket', + Key: 'file.txt', + Body: data, + StorageClass: 'STANDARD' // or 'STANDARD_IA' +})); +``` + +## Location Hints + +```bash +wrangler r2 bucket create my-bucket --location=enam + +# Hints: wnam, enam, weur, eeur, apac, oc +# Jurisdictions (override hint): --jurisdiction=eu (or fedramp) +``` + +## CORS Configuration + +CORS must be configured via S3 SDK or dashboard (not available in Workers API): + +```typescript +import { S3Client, PutBucketCorsCommand } from '@aws-sdk/client-s3'; + +const s3 = new S3Client({ + region: 'auto', + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + credentials: { + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY + } +}); + +await s3.send(new PutBucketCorsCommand({ + Bucket: 'my-bucket', + CORSConfiguration: { + CORSRules: [{ + AllowedOrigins: ['https://example.com'], + AllowedMethods: ['GET', 'PUT', 'HEAD'], + AllowedHeaders: ['*'], + ExposeHeaders: ['ETag'], + MaxAgeSeconds: 3600 + }] + } +})); +``` + +## Object Lifecycles + +```typescript +import { PutBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3'; + +await s3.send(new PutBucketLifecycleConfigurationCommand({ + Bucket: 'my-bucket', + LifecycleConfiguration: { + Rules: [ + { + ID: 'expire-old-logs', + Status: 'Enabled', + Prefix: 'logs/', + Expiration: { Days: 90 } + }, + { + ID: 'transition-to-ia', + Status: 'Enabled', + Prefix: 'archives/', + Transitions: [{ Days: 30, StorageClass: 'STANDARD_IA' }] + } + ] + } +})); +``` + +## API Token Scopes + +When creating R2 tokens, set minimal permissions: + +| Permission | Use Case | +|------------|----------| +| Object Read | Public serving, downloads | +| Object Write | Uploads only | +| Object Read & Write | Full object operations | +| Admin Read & Write | Bucket management, CORS, lifecycles | + +**Best practice:** Separate tokens for Workers (read/write) vs admin tasks (CORS, lifecycles). + +## Event Notifications + +```jsonc +// wrangler.jsonc +{ + "r2_buckets": [ + { + "binding": "MY_BUCKET", + "bucket_name": "my-bucket", + "event_notifications": [ + { + "queue": "r2-events", + "actions": ["PutObject", "DeleteObject", "CompleteMultipartUpload"] + } + ] + } + ], + "queues": { + "producers": [{ "binding": "R2_EVENTS", "queue": "r2-events" }], + "consumers": [{ "queue": "r2-events", "max_batch_size": 10 }] + } +} +``` + +## Bucket Management + +```bash +wrangler r2 bucket create my-bucket --location=enam --storage-class=Standard +wrangler r2 bucket list +wrangler r2 bucket info my-bucket +wrangler r2 bucket delete my-bucket # Must be empty +wrangler r2 bucket update-storage-class my-bucket --storage-class=InfrequentAccess + +# Public bucket via dashboard +wrangler r2 bucket domain add my-bucket --domain=files.example.com +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/r2/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/r2/gotchas.md new file mode 100644 index 0000000..ad755d3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2/gotchas.md @@ -0,0 +1,190 @@ +# R2 Gotchas & Troubleshooting + +## List Truncation + +```typescript +// ❌ WRONG: Don't compare object count when using include +while (listed.objects.length < options.limit) { ... } + +// ✅ CORRECT: Always use truncated property +while (listed.truncated) { + const next = await env.MY_BUCKET.list({ cursor: listed.cursor }); + // ... +} +``` + +**Reason:** `include` with metadata may return fewer objects per page to fit metadata. + +## ETag Format + +```typescript +// ❌ WRONG: Using etag (unquoted) in headers +headers.set('etag', object.etag); // Missing quotes + +// ✅ CORRECT: Use httpEtag (quoted) +headers.set('etag', object.httpEtag); +``` + +## Checksum Limits + +Only ONE checksum algorithm allowed per PUT: + +```typescript +// ❌ WRONG: Multiple checksums +await env.MY_BUCKET.put(key, data, { md5: hash1, sha256: hash2 }); // Error + +// ✅ CORRECT: Pick one +await env.MY_BUCKET.put(key, data, { sha256: hash }); +``` + +## Multipart Requirements + +- All parts must be uniform size (except last part) +- Part numbers start at 1 (not 0) +- Uncompleted uploads auto-abort after 7 days +- `resumeMultipartUpload` doesn't validate uploadId existence + +## Conditional Operations + +```typescript +// Precondition failure returns object WITHOUT body +const object = await env.MY_BUCKET.get(key, { + onlyIf: { etagMatches: '"wrong"' } +}); + +// Check for body, not just null +if (!object) return new Response('Not found', { status: 404 }); +if (!object.body) return new Response(null, { status: 304 }); // Precondition failed +``` + +## Key Validation + +```typescript +// ❌ DANGEROUS: Path traversal +const key = url.pathname.slice(1); // Could be ../../../etc/passwd +await env.MY_BUCKET.get(key); + +// ✅ SAFE: Validate keys +if (!key || key.includes('..') || key.startsWith('/')) { + return new Response('Invalid key', { status: 400 }); +} +``` + +## Storage Class Pitfalls + +- InfrequentAccess: 30-day minimum billing (even if deleted early) +- Can't transition IA → Standard via lifecycle (use S3 CopyObject) +- Retrieval fees apply for IA reads + +## Stream Length Requirement + +```typescript +// ❌ WRONG: Streaming unknown length fails silently +const response = await fetch(url); +await env.MY_BUCKET.put(key, response.body); // May fail without error + +// ✅ CORRECT: Buffer or use Content-Length +const data = await response.arrayBuffer(); +await env.MY_BUCKET.put(key, data); + +// OR: Pass Content-Length if known +const object = await env.MY_BUCKET.put(key, request.body, { + httpMetadata: { + contentLength: parseInt(request.headers.get('content-length') || '0') + } +}); +``` + +**Reason:** R2 requires known length for streams. Unknown length may cause silent truncation. + +## S3 SDK Region Configuration + +```typescript +// ❌ WRONG: Missing region breaks ALL S3 SDK calls +const s3 = new S3Client({ + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + credentials: { ... } +}); + +// ✅ CORRECT: MUST set region='auto' +const s3 = new S3Client({ + region: 'auto', // REQUIRED + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + credentials: { ... } +}); +``` + +**Reason:** S3 SDK requires region. R2 uses 'auto' as placeholder. + +## Local Development Limits + +```typescript +// ❌ Miniflare/wrangler dev: Limited R2 support +// - No multipart uploads +// - No presigned URLs (requires S3 SDK + network) +// - Memory-backed storage (lost on restart) + +// ✅ Use remote bindings for full features +wrangler dev --remote + +// OR: Conditional logic +if (env.ENVIRONMENT === 'development') { + // Fallback for local dev +} else { + // Full R2 features +} +``` + +## Presigned URL Expiry + +```typescript +// ❌ WRONG: URL expires but no client validation +const url = await getSignedUrl(s3, command, { expiresIn: 60 }); +// 61 seconds later: 403 Forbidden + +// ✅ CORRECT: Return expiry to client +return Response.json({ + uploadUrl: url, + expiresAt: new Date(Date.now() + 60000).toISOString() +}); +``` + +## Limits + +| Limit | Value | +|-------|-------| +| Object size | 5 TB | +| Multipart part count | 10,000 | +| Multipart part min size | 5 MB (except last) | +| Batch delete | 1,000 keys | +| List limit | 1,000 per request | +| Key size | 1024 bytes | +| Custom metadata | 2 KB per object | +| Presigned URL max expiry | 7 days | + +## Common Errors + +### "Stream upload failed" / Silent Truncation + +**Cause:** Stream length unknown or Content-Length missing +**Solution:** Buffer data or pass explicit Content-Length + +### "Invalid credentials" / S3 SDK + +**Cause:** Missing `region: 'auto'` in S3Client config +**Solution:** Always set `region: 'auto'` for R2 + +### "Object not found" + +**Cause:** Object key doesn't exist or was deleted +**Solution:** Verify object key correct, check if object was deleted, ensure bucket correct + +### "List compatibility error" + +**Cause:** Missing or old compatibility_date, or flag not enabled +**Solution:** Set `compatibility_date >= 2022-08-04` or enable `r2_list_honor_include` flag + +### "Multipart upload failed" + +**Cause:** Part sizes not uniform or incorrect part number +**Solution:** Ensure uniform size except final part, verify part numbers start at 1 diff --git a/plugins/cloudflare/skills/cloudflare/references/r2/patterns.md b/plugins/cloudflare/skills/cloudflare/references/r2/patterns.md new file mode 100644 index 0000000..85191d6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/r2/patterns.md @@ -0,0 +1,193 @@ +# R2 Patterns & Best Practices + +## Streaming Large Files + +```typescript +const object = await env.MY_BUCKET.get(key); +if (!object) return new Response('Not found', { status: 404 }); + +const headers = new Headers(); +object.writeHttpMetadata(headers); +headers.set('etag', object.httpEtag); + +return new Response(object.body, { headers }); +``` + +## Conditional GET (304 Not Modified) + +```typescript +const ifNoneMatch = request.headers.get('if-none-match'); +const object = await env.MY_BUCKET.get(key, { + onlyIf: { etagDoesNotMatch: ifNoneMatch?.replace(/"/g, '') || '' } +}); + +if (!object) return new Response('Not found', { status: 404 }); +if (!object.body) return new Response(null, { status: 304, headers: { 'etag': object.httpEtag } }); + +return new Response(object.body, { headers: { 'etag': object.httpEtag } }); +``` + +## Upload with Validation + +```typescript +const key = url.pathname.slice(1); +if (!key || key.includes('..')) return new Response('Invalid key', { status: 400 }); + +const object = await env.MY_BUCKET.put(key, request.body, { + httpMetadata: { contentType: request.headers.get('content-type') || 'application/octet-stream' }, + customMetadata: { uploadedAt: new Date().toISOString(), ip: request.headers.get('cf-connecting-ip') || 'unknown' } +}); + +return Response.json({ key: object.key, size: object.size, etag: object.httpEtag }); +``` + +## Multipart with Progress + +```typescript +const PART_SIZE = 5 * 1024 * 1024; // 5MB +const partCount = Math.ceil(file.size / PART_SIZE); +const multipart = await env.MY_BUCKET.createMultipartUpload(key, { httpMetadata: { contentType: file.type } }); + +const uploadedParts: R2UploadedPart[] = []; +try { + for (let i = 0; i < partCount; i++) { + const start = i * PART_SIZE; + const part = await multipart.uploadPart(i + 1, file.slice(start, start + PART_SIZE)); + uploadedParts.push(part); + onProgress?.(Math.round(((i + 1) / partCount) * 100)); + } + return await multipart.complete(uploadedParts); +} catch (error) { + await multipart.abort(); + throw error; +} +``` + +## Batch Delete + +```typescript +async function deletePrefix(prefix: string, env: Env) { + let cursor: string | undefined; + let truncated = true; + + while (truncated) { + const listed = await env.MY_BUCKET.list({ prefix, limit: 1000, cursor }); + if (listed.objects.length > 0) { + await env.MY_BUCKET.delete(listed.objects.map(o => o.key)); + } + truncated = listed.truncated; + cursor = listed.cursor; + } +} +``` + +## Checksum Validation & Storage Transitions + +```typescript +// Upload with checksum +const hash = await crypto.subtle.digest('SHA-256', data); +await env.MY_BUCKET.put(key, data, { sha256: hash }); + +// Transition storage class (requires S3 SDK) +import { S3Client, CopyObjectCommand } from '@aws-sdk/client-s3'; +await s3.send(new CopyObjectCommand({ + Bucket: 'my-bucket', Key: key, + CopySource: `/my-bucket/${key}`, + StorageClass: 'STANDARD_IA' +})); +``` + +## Client-Side Uploads (Presigned URLs) + +```typescript +import { S3Client } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; + +// Worker: Generate presigned upload URL +const s3 = new S3Client({ + region: 'auto', + endpoint: `https://${env.ACCOUNT_ID}.r2.cloudflarestorage.com`, + credentials: { accessKeyId: env.R2_ACCESS_KEY_ID, secretAccessKey: env.R2_SECRET_ACCESS_KEY } +}); + +const url = await getSignedUrl(s3, new PutObjectCommand({ Bucket: 'my-bucket', Key: key }), { expiresIn: 3600 }); +return Response.json({ uploadUrl: url }); + +// Client: Upload directly +const { uploadUrl } = await fetch('/api/upload-url').then(r => r.json()); +await fetch(uploadUrl, { method: 'PUT', body: file }); +``` + +## Caching with Cache API + +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const cache = caches.default; + const url = new URL(request.url); + const cacheKey = new Request(url.toString(), request); + + // Check cache first + let response = await cache.match(cacheKey); + if (response) return response; + + // Fetch from R2 + const key = url.pathname.slice(1); + const object = await env.MY_BUCKET.get(key); + if (!object) return new Response('Not found', { status: 404 }); + + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set('etag', object.httpEtag); + headers.set('cache-control', 'public, max-age=31536000, immutable'); + + response = new Response(object.body, { headers }); + + // Cache for subsequent requests + ctx.waitUntil(cache.put(cacheKey, response.clone())); + + return response; + } +}; +``` + +## Public Bucket with Custom Domain + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + // CORS preflight + if (request.method === 'OPTIONS') { + return new Response(null, { + headers: { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET, HEAD', + 'access-control-max-age': '86400' + } + }); + } + + const key = new URL(request.url).pathname.slice(1); + if (!key) return Response.redirect('/index.html', 302); + + const object = await env.MY_BUCKET.get(key); + if (!object) return new Response('Not found', { status: 404 }); + + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set('etag', object.httpEtag); + headers.set('access-control-allow-origin', '*'); + headers.set('cache-control', 'public, max-age=31536000, immutable'); + + return new Response(object.body, { headers }); + } +}; +``` + +## r2.dev Public URLs + +Enable r2.dev in dashboard for simple public access: `https://pub-${hashId}.r2.dev/${key}` +Or add custom domain via dashboard: `https://files.example.com/${key}` + +**Limitations:** No auth, bucket-level CORS, no cache override. diff --git a/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/README.md b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/README.md new file mode 100644 index 0000000..6f99921 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/README.md @@ -0,0 +1,65 @@ +# Cloudflare Realtime SFU Reference + +Expert guidance for building real-time audio/video/data applications using Cloudflare Realtime SFU (Selective Forwarding Unit). + +## Reading Order + +| Task | Files | ~Tokens | +|------|-------|---------| +| New project | README → configuration | ~1200 | +| Implement publish/subscribe | README → api | ~1600 | +| Add PartyTracks | patterns (PartyTracks section) | ~800 | +| Build presence system | patterns (DO section) | ~800 | +| Debug connection issues | gotchas | ~700 | +| Scale to millions | patterns (Cascading section) | ~600 | +| Add simulcast | patterns (Advanced section) | ~500 | +| Configure TURN | configuration (TURN section) | ~400 | + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, environment variables, Wrangler config +- **[api.md](api.md)** - Sessions, tracks, endpoints, request/response patterns +- **[patterns.md](patterns.md)** - Architecture patterns, use cases, integration examples +- **[gotchas.md](gotchas.md)** - Common issues, debugging, performance, security + +## Quick Start + +Cloudflare Realtime SFU: WebRTC infrastructure on global network (310+ cities). Anycast routing, no regional constraints, pub/sub model. + +**Core concepts:** +- **Sessions:** WebRTC PeerConnection to Cloudflare edge +- **Tracks:** Audio/video/data channels you publish or subscribe to +- **No rooms:** Build presence layer yourself via track sharing (see patterns.md) + +**Mental model:** Your client establishes one WebRTC session, publishes tracks (audio/video), shares track IDs via your backend, others subscribe to your tracks using track IDs + your session ID. + +## Choose Your Approach + +| Approach | When to Use | Complexity | +|----------|-------------|------------| +| **PartyTracks** | Production apps with device switching, React | Low - Observable-based, handles reconnections | +| **Raw API** | Custom requirements, non-browser, learning | Medium - Full control, manual WebRTC lifecycle | +| **RealtimeKit** | End-to-end SDK with UI components | Lowest - Managed state, React hooks | + +**Recommendation:** Start with PartyTracks for most production applications. See patterns.md for PartyTracks examples. + +## SFU vs RealtimeKit + +- **Realtime SFU:** WebRTC infrastructure (this reference). Build your own signaling, presence, UI. +- **RealtimeKit:** SDK layer on top of SFU. Includes React hooks, state management, UI components. Part of Cloudflare AI platform. + +Use SFU directly when you need custom signaling or non-React framework. Use RealtimeKit for faster development with React. + +## Setup + +Dashboard: https://dash.cloudflare.com/?to=/:account/calls + +Get `CALLS_APP_ID` and `CALLS_APP_SECRET` from dashboard, then see configuration.md for deployment. + +## See Also + +- [Orange Meets Demo](https://demo.orange.cloudflare.dev/) +- [Orange Source](https://github.com/cloudflare/orange) +- [Calls Examples](https://github.com/cloudflare/calls-examples) +- [API Reference](https://developers.cloudflare.com/api/resources/calls/) +- [RealtimeKit Docs](https://developers.cloudflare.com/workers-ai/realtimekit/) diff --git a/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/api.md b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/api.md new file mode 100644 index 0000000..6e6dae6 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/api.md @@ -0,0 +1,158 @@ +# API Reference + +## Authentication + +```bash +curl -X POST 'https://rtc.live/v1/apps/${CALLS_APP_ID}/sessions/new' \ + -H "Authorization: Bearer ${CALLS_APP_SECRET}" +``` + +## Core Concepts + +**Sessions:** PeerConnection to Cloudflare edge +**Tracks:** Media/data channels (audio/video/datachannel) +**No rooms:** Build presence via track sharing + +## Client Libraries + +**PartyTracks (Recommended):** Observable-based client library for production use. Handles device changes, network switches, ICE restarts automatically. Push/pull API with React hooks. See patterns.md for full examples. + +```bash +npm install partytracks @cloudflare/calls +``` + +**Raw API:** Direct HTTP + WebRTC for custom requirements (documented below). + +## Endpoints + +### Create Session +```http +POST /v1/apps/{appId}/sessions/new +→ {sessionId, sessionDescription} +``` + +### Add Track (Publish) +```http +POST /v1/apps/{appId}/sessions/{sessionId}/tracks/new +Body: { + sessionDescription: {sdp, type: "offer"}, + tracks: [{location: "local", trackName: "my-video"}] +} +→ {sessionDescription, tracks: [{trackName}]} +``` + +### Add Track (Subscribe) +```http +POST /v1/apps/{appId}/sessions/{sessionId}/tracks/new +Body: { + tracks: [{ + location: "remote", + trackName: "remote-track-id", + sessionId: "other-session-id" + }] +} +→ {sessionDescription} (server offer) +``` + +### Renegotiate +```http +PUT /v1/apps/{appId}/sessions/{sessionId}/renegotiate +Body: {sessionDescription: {sdp, type: "answer"}} +``` + +### Close Tracks +```http +PUT /v1/apps/{appId}/sessions/{sessionId}/tracks/close +Body: {tracks: [{trackName}]} +→ {requiresImmediateRenegotiation: boolean} +``` + +### Get Session +```http +GET /v1/apps/{appId}/sessions/{sessionId} +→ {sessionId, tracks: TrackMetadata[]} +``` + +## TypeScript Types + +```typescript +interface TrackMetadata { + trackName: string; + location: "local" | "remote"; + sessionId?: string; // For remote tracks + mid?: string; // WebRTC mid +} +``` + +## WebRTC Flow + +```typescript +// 1. Create PeerConnection +const pc = new RTCPeerConnection({ + iceServers: [{urls: 'stun:stun.cloudflare.com:3478'}] +}); + +// 2. Add tracks +const stream = await navigator.mediaDevices.getUserMedia({video: true, audio: true}); +stream.getTracks().forEach(track => pc.addTrack(track, stream)); + +// 3. Create offer +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +// 4. Send to backend → Cloudflare API +const response = await fetch('/api/new-session', { + method: 'POST', + body: JSON.stringify({sdp: offer.sdp}) +}); + +// 5. Set remote answer +const {sessionDescription} = await response.json(); +await pc.setRemoteDescription(sessionDescription); +``` + +## Publishing + +```typescript +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +const res = await fetch(`/api/sessions/${sessionId}/tracks`, { + method: 'POST', + body: JSON.stringify({ + sdp: offer.sdp, + tracks: [{location: 'local', trackName: 'my-video'}] + }) +}); + +const {sessionDescription, tracks} = await res.json(); +await pc.setRemoteDescription(sessionDescription); +const publishedTrackId = tracks[0].trackName; // Share with others +``` + +## Subscribing + +```typescript +const res = await fetch(`/api/sessions/${sessionId}/tracks`, { + method: 'POST', + body: JSON.stringify({ + tracks: [{location: 'remote', trackName: remoteTrackId, sessionId: remoteSessionId}] + }) +}); + +const {sessionDescription} = await res.json(); +await pc.setRemoteDescription(sessionDescription); + +const answer = await pc.createAnswer(); +await pc.setLocalDescription(answer); + +await fetch(`/api/sessions/${sessionId}/renegotiate`, { + method: 'PUT', + body: JSON.stringify({sdp: answer.sdp}) +}); + +pc.ontrack = (event) => { + const [remoteStream] = event.streams; + videoElement.srcObject = remoteStream; +}; +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/configuration.md b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/configuration.md new file mode 100644 index 0000000..6736b45 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/configuration.md @@ -0,0 +1,137 @@ +# Configuration & Deployment + +## Dashboard Setup + +1. Navigate to https://dash.cloudflare.com/?to=/:account/calls +2. Click "Create Application" (or use existing app) +3. Copy `CALLS_APP_ID` from dashboard +4. Generate and copy `CALLS_APP_SECRET` (treat as sensitive credential) +5. Use credentials in Wrangler config or environment variables below + +## Dependencies + +**Backend (Workers):** Built-in fetch API, no additional packages required + +**Client (PartyTracks):** +```bash +npm install partytracks @cloudflare/calls +``` + +**Client (React + PartyTracks):** +```bash +npm install partytracks @cloudflare/calls observable-hooks +# Observable hooks: useObservableAsValue, useValueAsObservable +``` + +**Client (Raw API):** Native browser WebRTC API only + +## Wrangler Setup + +```jsonc +{ + "name": "my-calls-app", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + "vars": { + "CALLS_APP_ID": "your-app-id", + "MAX_WEBCAM_BITRATE": "1200000", + "MAX_WEBCAM_FRAMERATE": "24", + "MAX_WEBCAM_QUALITY_LEVEL": "1080" + }, + // Set secret: wrangler secret put CALLS_APP_SECRET + "durable_objects": { + "bindings": [ + { + "name": "ROOM", + "class_name": "Room" + } + ] + } +} +``` + +## Deploy + +```bash +wrangler login +wrangler secret put CALLS_APP_SECRET +wrangler deploy +``` + +## Environment Variables + +**Required:** +- `CALLS_APP_ID`: From dashboard +- `CALLS_APP_SECRET`: From dashboard (secret) + +**Optional:** +- `MAX_WEBCAM_BITRATE` (default: 1200000) +- `MAX_WEBCAM_FRAMERATE` (default: 24) +- `MAX_WEBCAM_QUALITY_LEVEL` (default: 1080) +- `TURN_SERVICE_ID`: TURN service +- `TURN_SERVICE_TOKEN`: TURN auth (secret) + +## TURN Configuration + +```javascript +const pc = new RTCPeerConnection({ + iceServers: [ + { urls: 'stun:stun.cloudflare.com:3478' }, + { + urls: [ + 'turn:turn.cloudflare.com:3478?transport=udp', + 'turn:turn.cloudflare.com:3478?transport=tcp', + 'turns:turn.cloudflare.com:5349?transport=tcp' + ], + username: turnUsername, + credential: turnCredential + } + ], + bundlePolicy: 'max-bundle', // Recommended: reduces overhead + iceTransportPolicy: 'all' // Use 'relay' to force TURN (testing only) +}); +``` + +**Ports:** 3478 (UDP/TCP), 53 (UDP), 80 (TCP), 443 (TLS), 5349 (TLS) + +**When to use TURN:** Required for restrictive corporate firewalls/networks that block UDP. ~5-10% of connections fallback to TURN. STUN works for most users. + +**ICE candidate filtering:** Cloudflare handles candidate filtering automatically. No need to manually filter candidates. + +## Durable Object Boilerplate + +Minimal presence system: + +```typescript +export class Room { + private sessions = new Map(); + + async fetch(req: Request) { + const {pathname} = new URL(req.url); + const body = await req.json(); + + if (pathname === '/join') { + this.sessions.set(body.sessionId, {userId: body.userId, tracks: []}); + return Response.json({participants: this.sessions.size}); + } + + if (pathname === '/publish') { + this.sessions.get(body.sessionId)?.tracks.push(...body.tracks); + // Broadcast to others via WebSocket (not shown) + return new Response('OK'); + } + + return new Response('Not found', {status: 404}); + } +} +``` + +## Environment Validation + +Check credentials before first API call: + +```typescript +if (!env.CALLS_APP_ID || !env.CALLS_APP_SECRET) { + throw new Error('CALLS_APP_ID and CALLS_APP_SECRET required'); +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/gotchas.md new file mode 100644 index 0000000..efe5ee7 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/gotchas.md @@ -0,0 +1,133 @@ +# Gotchas & Troubleshooting + +## Common Errors + +### "Slow initial connect (~1.8s)" + +**Cause:** First STUN delayed during consensus forming (normal behavior) +**Solution:** Subsequent connections are faster. CF detects DTLS ClientHello early to compensate. + +### "No media flow" + +**Cause:** SDP exchange incomplete, connection not established, tracks not added before offer, browser permissions missing +**Solution:** +1. Verify SDP exchange complete +2. Check `pc.connectionState === 'connected'` +3. Ensure tracks added before creating offer +4. Confirm browser permissions granted +5. Use `chrome://webrtc-internals` for debugging + +### "Track not receiving" + +**Cause:** Track not published, track ID not shared, session IDs mismatch, `pc.ontrack` not set, renegotiation needed +**Solution:** +1. Verify track published successfully +2. Confirm track ID shared between peers +3. Check session IDs match +4. Set `pc.ontrack` handler before answer +5. Trigger renegotiation if needed + +### "ICE connection failed" + +**Cause:** Network changed, firewall blocked UDP, TURN needed, transient network issue +**Solution:** +```typescript +pc.oniceconnectionstatechange = async () => { + if (pc.iceConnectionState === 'failed') { + console.warn('ICE failed, attempting restart'); + await pc.restartIce(); // Triggers new ICE gathering + + // Create new offer with ICE restart flag + const offer = await pc.createOffer({iceRestart: true}); + await pc.setLocalDescription(offer); + + // Send to backend → Cloudflare API + await fetch(`/api/sessions/${sessionId}/renegotiate`, { + method: 'PUT', + body: JSON.stringify({sdp: offer.sdp}) + }); + } +}; +``` + +### "Track stuck/frozen" + +**Cause:** Sender paused track, network congestion, codec mismatch, mobile browser backgrounded +**Solution:** +1. Check `track.enabled` and `track.readyState === 'live'` +2. Verify sender active: `pc.getSenders().find(s => s.track === track)` +3. Check stats for packet loss/jitter (see patterns.md) +4. On mobile: Re-acquire tracks when app foregrounded +5. Test with different codecs if persistent + +### "Network change disconnects call" + +**Cause:** Mobile switching WiFi↔cellular, laptop changing networks +**Solution:** +```typescript +// Listen for network changes +if ('connection' in navigator) { + (navigator as any).connection.addEventListener('change', async () => { + console.log('Network changed'); + await pc.restartIce(); // Use ICE restart pattern above + }); +} + +// Or use PartyTracks (handles automatically) +``` + +## Retry with Exponential Backoff + +```typescript +async function fetchWithRetry(url: string, options: RequestInit, maxRetries = 3) { + for (let i = 0; i < maxRetries; i++) { + try { + const res = await fetch(url, options); + if (res.ok) return res; + if (res.status >= 500) throw new Error('Server error'); + return res; // Client error, don't retry + } catch (err) { + if (i === maxRetries - 1) throw err; + const delay = Math.min(1000 * 2 ** i, 10000); // Cap at 10s + await new Promise(resolve => setTimeout(resolve, delay)); + } + } +} +``` + +## Debugging with chrome://webrtc-internals + +1. Open `chrome://webrtc-internals` in Chrome/Edge +2. Find your PeerConnection in the list +3. Check **Stats graphs** for packet loss, jitter, bandwidth +4. Check **ICE candidate pairs**: Look for `succeeded` state, relay vs host candidates +5. Check **getStats**: Raw metrics for inbound/outbound RTP +6. Look for errors in **Event log**: `iceConnectionState`, `connectionState` changes +7. Export data with "Download the PeerConnection updates and stats data" button +8. Common issues visible here: ICE failures, high packet loss, bitrate drops + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Egress (Free) | 1TB/month | Per account | +| Egress (Paid) | $0.05/GB | After free tier | +| Inbound traffic | Free | All plans | +| TURN service | Free | Included with SFU | +| Participants | No hard limit | Client bandwidth/CPU bound (typically 10-50 tracks) | +| Tracks per session | No hard limit | Client resources limited | +| Session duration | No hard limit | Production calls run for hours | +| WebRTC ports | UDP 1024-65535 | Outbound only, required for media | +| API rate limit | 600 req/min | Per app, burst allowed | + +## Security Checklist + +- ✅ **Never expose** `CALLS_APP_SECRET` to client +- ✅ **Validate user identity** in backend before creating sessions +- ✅ **Implement auth tokens** for session access (JWT in custom header) +- ✅ **Rate limit** session creation endpoints +- ✅ **Expire sessions** server-side after inactivity +- ✅ **Validate track IDs** before subscribing (prevent unauthorized access) +- ✅ **Use HTTPS** for all signaling (API calls) +- ✅ **Enable DTLS-SRTP** (automatic with Cloudflare, encrypts media) +- ⚠️ **Consider E2EE** for sensitive content (implement client-side with Insertable Streams API) diff --git a/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/patterns.md b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/patterns.md new file mode 100644 index 0000000..95ddc42 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtime-sfu/patterns.md @@ -0,0 +1,174 @@ +# Patterns & Use Cases + +## Architecture + +``` +Client (WebRTC) <---> CF Edge <---> Backend (HTTP) + | + CF Backbone (310+ DCs) + | + Other Edges <---> Other Clients +``` + +Anycast: Last-mile <50ms (95%), no region select, NACK shield, distributed consensus + +Cascading trees auto-scale to millions: +``` +Publisher -> Edge A -> Edge B -> Sub1 + \-> Edge C -> Sub2,3 +``` + +## Use Cases + +**1:1:** A creates session+publishes, B creates+subscribes to A+publishes, A subscribes to B +**N:N:** All create session+publish, backend broadcasts track IDs, all subscribe to others +**1:N:** Publisher creates+publishes, viewers each create+subscribe (no fan-out limit) +**Breakout:** Same PeerConnection! Backend closes/adds tracks, no recreation + +## PartyTracks (Recommended) + +Observable-based client with automatic device/network handling: + +```typescript +import {PartyTracks} from 'partytracks'; + +// Create client +const pt = new PartyTracks({ + apiUrl: '/api/calls', + sessionId: 'my-session', + onTrack: (track, peer) => { + const video = document.getElementById(`video-${peer.id}`) as HTMLVideoElement; + video.srcObject = new MediaStream([track]); + } +}); + +// Publish camera (push API) +const camera = await pt.getCamera(); // Auto-requests permissions, handles device changes +await pt.publishTrack(camera, {trackName: 'my-camera'}); + +// Subscribe to remote track (pull API) +await pt.subscribeToTrack({trackName: 'remote-camera', sessionId: 'other-session'}); + +// React hook example +import {useObservableAsValue} from 'observable-hooks'; + +function VideoCall() { + const localTracks = useObservableAsValue(pt.localTracks$); + const remoteTracks = useObservableAsValue(pt.remoteTracks$); + + return
{/* Render tracks */}
; +} + +// Screenshare +const screen = await pt.getScreenshare(); +await pt.publishTrack(screen, {trackName: 'my-screen'}); + +// Handle device changes (automatic) +// PartyTracks detects device changes (e.g., Bluetooth headset) and renegotiates +``` + +## Backend + +Express: +```js +app.post('/api/new-session', async (req, res) => { + const r = await fetch(`${CALLS_API}/apps/${process.env.CALLS_APP_ID}/sessions/new`, + {method: 'POST', headers: {'Authorization': `Bearer ${process.env.CALLS_APP_SECRET}`}}); + res.json(await r.json()); +}); +``` + +Workers: Same pattern, use `env.CALLS_APP_ID` and `env.CALLS_APP_SECRET` + +DO Presence: See configuration.md for boilerplate + +## Audio Level Detection + +```typescript +// Attach analyzer to audio track +function attachAudioLevelDetector(track: MediaStreamTrack) { + const ctx = new AudioContext(); + const analyzer = ctx.createAnalyser(); + const src = ctx.createMediaStreamSource(new MediaStream([track])); + src.connect(analyzer); + + const data = new Uint8Array(analyzer.frequencyBinCount); + const checkLevel = () => { + analyzer.getByteFrequencyData(data); + const level = data.reduce((a, b) => a + b) / data.length; + if (level > 30) console.log('Speaking:', level); // Trigger UI update + requestAnimationFrame(checkLevel); + }; + checkLevel(); +} +``` + +## Connection Quality Monitoring + +```typescript +pc.getStats().then(stats => { + stats.forEach(report => { + if (report.type === 'inbound-rtp' && report.kind === 'video') { + const {packetsLost, packetsReceived, jitter} = report; + const lossRate = packetsLost / (packetsLost + packetsReceived); + if (lossRate > 0.05) console.warn('High packet loss:', lossRate); + if (jitter > 100) console.warn('High jitter:', jitter); + } + }); +}); +``` + +## Stage Management (Limit Visible Participants) + +```typescript +// Subscribe to top 6 active speakers only +let activeSubscriptions = new Set(); + +function updateStage(topSpeakers: string[]) { + const toAdd = topSpeakers.filter(id => !activeSubscriptions.has(id)).slice(0, 6); + const toRemove = [...activeSubscriptions].filter(id => !topSpeakers.includes(id)); + + toRemove.forEach(id => { + pc.getSenders().find(s => s.track?.id === id)?.track?.stop(); + activeSubscriptions.delete(id); + }); + + toAdd.forEach(async id => { + await fetch(`/api/subscribe`, {method: 'POST', body: JSON.stringify({trackId: id})}); + activeSubscriptions.add(id); + }); +} +``` + +## Advanced + +Bandwidth mgmt: +```ts +const s = pc.getSenders().find(s => s.track?.kind === 'video'); +const p = s.getParameters(); +if (!p.encodings) p.encodings = [{}]; +p.encodings[0].maxBitrate = 1200000; p.encodings[0].maxFramerate = 24; +await s.setParameters(p); +``` + +Simulcast (CF auto-forwards best layer): +```ts +pc.addTransceiver('video', {direction: 'sendonly', sendEncodings: [ + {rid: 'high', maxBitrate: 1200000}, + {rid: 'med', maxBitrate: 600000, scaleResolutionDownBy: 2}, + {rid: 'low', maxBitrate: 200000, scaleResolutionDownBy: 4} +]}); +``` + +DataChannel: +```ts +const dc = pc.createDataChannel('chat', {ordered: true, maxRetransmits: 3}); +dc.onopen = () => dc.send(JSON.stringify({type: 'chat', text: 'Hi'})); +dc.onmessage = (e) => console.log('RX:', JSON.parse(e.data)); +``` + +**WHIP/WHEP:** For streaming interop (OBS → SFU, SFU → video players), use WHIP (ingest) and WHEP (egress) protocols. See Cloudflare Stream integration docs. + +Integrations: R2 for recording `env.R2_BUCKET.put(...)`, Queues for analytics + +Perf: 100-250ms connect, ~50ms latency (95%), 200-400ms glass-to-glass, no participant limit (client: 10-50 tracks) diff --git a/plugins/cloudflare/skills/cloudflare/references/realtimekit/README.md b/plugins/cloudflare/skills/cloudflare/references/realtimekit/README.md new file mode 100644 index 0000000..6d19f51 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtimekit/README.md @@ -0,0 +1,113 @@ +# Cloudflare RealtimeKit + +Expert guidance for building real-time video and audio applications using **Cloudflare RealtimeKit** - a comprehensive SDK suite for adding customizable live video and voice to web or mobile applications. + +## Overview + +RealtimeKit is Cloudflare's SDK suite built on Realtime SFU, abstracting WebRTC complexity with fast integration, pre-built UI components, global performance (300+ cities), and production features (recording, transcription, chat, polls). + +**Use cases**: Team meetings, webinars, social video, audio calls, interactive plugins + +## Core Concepts + +- **App**: Workspace grouping meetings, participants, presets, recordings. Use separate Apps for staging/production +- **Meeting**: Re-usable virtual room. Each join creates new **Session** +- **Session**: Live meeting instance. Created on first join, ends after last leave +- **Participant**: User added via REST API. Returns `authToken` for client SDK. **Do not reuse tokens** +- **Preset**: Reusable permission/UI template (permissions, meeting type, theme). Applied at participant creation +- **Peer ID** (`id`): Unique per session, changes on rejoin +- **Participant ID** (`userId`): Persistent across sessions + +## Quick Start + +### 1. Create App & Meeting (Backend) + +```bash +# Create app +curl -X POST 'https://api.cloudflare.com/client/v4/accounts//realtime/kit/apps' \ + -H 'Authorization: Bearer ' \ + -d '{"name": "My RealtimeKit App"}' + +# Create meeting +curl -X POST 'https://api.cloudflare.com/client/v4/accounts//realtime/kit//meetings' \ + -H 'Authorization: Bearer ' \ + -d '{"title": "Team Standup"}' + +# Add participant +curl -X POST 'https://api.cloudflare.com/client/v4/accounts//realtime/kit//meetings//participants' \ + -H 'Authorization: Bearer ' \ + -d '{"name": "Alice", "preset_name": "host"}' +# Returns: { authToken } +``` + +### 2. Client Integration + +**React**: +```tsx +import { RtkMeeting } from '@cloudflare/realtimekit-react-ui'; + +function App() { + return {}} />; +} +``` + +**Core SDK**: +```typescript +import RealtimeKitClient from '@cloudflare/realtimekit'; + +const meeting = new RealtimeKitClient({ authToken: '', video: true, audio: true }); +await meeting.join(); +``` + +## Reading Order + +| Task | Files | +|------|-------| +| Quick integration | README only | +| Custom UI | README → patterns → api | +| Backend setup | README → configuration | +| Debug issues | gotchas | +| Advanced features | patterns → api | + +## RealtimeKit vs Realtime SFU + +| Choose | When | +|--------|------| +| **RealtimeKit** | Need pre-built UI, fast integration, React/Angular/HTML | +| **Realtime SFU** | Building from scratch, custom WebRTC, full control | + +RealtimeKit is built on Realtime SFU but abstracts WebRTC complexity with UI components and SDKs. + +## Which Package? + +Need pre-built meeting UI? +- React → `@cloudflare/realtimekit-react-ui` (``) +- Angular → `@cloudflare/realtimekit-angular-ui` +- HTML/Vanilla → `@cloudflare/realtimekit-ui` + +Need custom UI? +- Core SDK → `@cloudflare/realtimekit` (RealtimeKitClient) - full control + +Need raw WebRTC control? +- See `realtime-sfu/` reference + +## In This Reference + +- [Configuration](./configuration.md) - Setup, installation, wrangler config +- [API](./api.md) - Meeting object, REST API, SDK methods +- [Patterns](./patterns.md) - Common workflows, code examples +- [Gotchas](./gotchas.md) - Common issues, troubleshooting + +## See Also + +- [Workers](../workers/) - Backend integration +- [D1](../d1/) - Meeting metadata storage +- [R2](../r2/) - Recording storage +- [KV](../kv/) - Session management + +## Reference Links + +- **Official Docs**: https://developers.cloudflare.com/realtime/realtimekit/ +- **API Reference**: https://developers.cloudflare.com/api/resources/realtime_kit/ +- **Examples**: https://github.com/cloudflare/realtimekit-web-examples +- **Dashboard**: https://dash.cloudflare.com/?to=/:account/realtime/kit diff --git a/plugins/cloudflare/skills/cloudflare/references/realtimekit/api.md b/plugins/cloudflare/skills/cloudflare/references/realtimekit/api.md new file mode 100644 index 0000000..18e9a3f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtimekit/api.md @@ -0,0 +1,212 @@ +# RealtimeKit API Reference + +Complete API reference for Meeting object, REST endpoints, and SDK methods. + +## Meeting Object API + +### `meeting.self` - Local Participant + +```typescript +// Properties: id, userId, name, audioEnabled, videoEnabled, screenShareEnabled, audioTrack, videoTrack, screenShareTracks, roomJoined, roomState +// Methods +await meeting.self.enableAudio() / disableAudio() / enableVideo() / disableVideo() / enableScreenShare() / disableScreenShare() +await meeting.self.setName("Name") // Before join only +await meeting.self.setDevice(device) +const devices = await meeting.self.getAllDevices() / getAudioDevices() / getVideoDevices() / getSpeakerDevices() +// Events: 'roomJoined', 'audioUpdate', 'videoUpdate', 'screenShareUpdate', 'deviceUpdate', 'deviceListUpdate' +meeting.self.on('roomJoined', () => {}) +meeting.self.on('audioUpdate', ({ audioEnabled, audioTrack }) => {}) +``` + +### `meeting.participants` - Remote Participants + +**Collections**: +```typescript +meeting.participants.joined / active / waitlisted / pinned // Maps +const participants = meeting.participants.joined.toArray() +const count = meeting.participants.joined.size() +const p = meeting.participants.joined.get('peer-id') +``` + +**Participant Properties**: +```typescript +participant.id / userId / name +participant.audioEnabled / videoEnabled / screenShareEnabled +participant.audioTrack / videoTrack / screenShareTracks +``` + +**Events**: +```typescript +meeting.participants.joined.on('participantJoined', (participant) => {}) +meeting.participants.joined.on('participantLeft', (participant) => {}) +``` + +### `meeting.meta` - Metadata +```typescript +meeting.meta.meetingId / meetingTitle / meetingStartedTimestamp +``` + +### `meeting.chat` - Chat +```typescript +meeting.chat.messages // Array +await meeting.chat.sendTextMessage("Hello") / sendImageMessage(file) +meeting.chat.on('chatUpdate', ({ message, messages }) => {}) +``` + +### `meeting.polls` - Polling +```typescript +meeting.polls.items // Array +await meeting.polls.create(question, options, anonymous, hideVotes) +await meeting.polls.vote(pollId, optionIndex) +``` + +### `meeting.plugins` - Collaborative Apps +```typescript +meeting.plugins.all // Array +await meeting.plugins.activate(pluginId) / deactivate() +``` + +### `meeting.ai` - AI Features +```typescript +meeting.ai.transcripts // Live transcriptions (when enabled in Preset) +``` + +### Core Methods +```typescript +await meeting.join() // Emits 'roomJoined' on meeting.self +await meeting.leave() +``` + +## TypeScript Types + +```typescript +import type { RealtimeKitClient, States, UIConfig, Participant } from '@cloudflare/realtimekit'; + +// Main interface +interface RealtimeKitClient { + self: SelfState; // Local participant (id, userId, name, audioEnabled, videoEnabled, roomJoined, roomState) + participants: { joined, active, waitlisted, pinned }; // Reactive Maps + chat: ChatNamespace; // messages[], sendTextMessage(), sendImageMessage() + polls: PollsNamespace; // items[], create(), vote() + plugins: PluginsNamespace; // all[], activate(), deactivate() + ai: AINamespace; // transcripts[] + meta: MetaState; // meetingId, meetingTitle, meetingStartedTimestamp + join(): Promise; + leave(): Promise; +} + +// Participant (self & remote share same shape) +interface Participant { + id: string; // Peer ID (changes on rejoin) + userId: string; // Persistent participant ID + name: string; + audioEnabled: boolean; + videoEnabled: boolean; + screenShareEnabled: boolean; + audioTrack: MediaStreamTrack | null; + videoTrack: MediaStreamTrack | null; + screenShareTracks: MediaStreamTrack[]; +} +``` + +## Store Architecture + +RealtimeKit uses reactive store (event-driven updates, live Maps): + +```typescript +// Subscribe to state changes +meeting.self.on('audioUpdate', ({ audioEnabled, audioTrack }) => {}); +meeting.participants.joined.on('participantJoined', (p) => {}); + +// Access current state synchronously +const isAudioOn = meeting.self.audioEnabled; +const count = meeting.participants.joined.size(); +``` + +**Key principles:** State updates emit events after changes. Use `.toArray()` sparingly. Collections are live Maps. + +## REST API + +Base: `https://api.cloudflare.com/client/v4/accounts/{account_id}/realtime/kit/{app_id}` + +### Meetings +```bash +GET /meetings # List all +GET /meetings/{meeting_id} # Get details +POST /meetings # Create: {"title": "..."} +PATCH /meetings/{meeting_id} # Update: {"title": "...", "record_on_start": true} +``` + +### Participants +```bash +GET /meetings/{meeting_id}/participants # List all +GET /meetings/{meeting_id}/participants/{participant_id} # Get details +POST /meetings/{meeting_id}/participants # Add: {"name": "...", "preset_name": "...", "custom_participant_id": "..."} +PATCH /meetings/{meeting_id}/participants/{participant_id} # Update: {"name": "...", "preset_name": "..."} +DELETE /meetings/{meeting_id}/participants/{participant_id} # Delete +POST /meetings/{meeting_id}/participants/{participant_id}/token # Refresh token +``` + +### Active Session +```bash +GET /meetings/{meeting_id}/active-session # Get active session +POST /meetings/{meeting_id}/active-session/kick # Kick users: {"user_ids": ["id1", "id2"]} +POST /meetings/{meeting_id}/active-session/kick-all # Kick all +POST /meetings/{meeting_id}/active-session/poll # Create poll: {"question": "...", "options": [...], "anonymous": false} +``` + +### Recording +```bash +GET /recordings?meeting_id={meeting_id} # List recordings +GET /recordings/active-recording/{meeting_id} # Get active recording +POST /recordings # Start: {"meeting_id": "...", "type": "composite"} (or "track") +PUT /recordings/{recording_id} # Control: {"action": "pause"} (or "resume", "stop") +POST /recordings/track # Track recording: {"meeting_id": "...", "layers": [...]} +``` + +### Livestreaming +```bash +GET /livestreams?exclude_meetings=false # List all +GET /livestreams/{livestream_id} # Get details +POST /meetings/{meeting_id}/livestreams # Start for meeting +POST /meetings/{meeting_id}/active-livestream/stop # Stop +POST /livestreams # Create independent: returns {ingest_server, stream_key, playback_url} +``` + +### Sessions & Analytics +```bash +GET /sessions # List all +GET /sessions/{session_id} # Get details +GET /sessions/{session_id}/participants # List participants +GET /sessions/{session_id}/participants/{participant_id} # Call stats +GET /sessions/{session_id}/chat # Download chat CSV +GET /sessions/{session_id}/transcript # Download transcript CSV +GET /sessions/{session_id}/summary # Get summary +POST /sessions/{session_id}/summary # Generate summary +GET /analytics/daywise?start_date=YYYY-MM-DD&end_date=YYYY-MM-DD # Day-wise analytics +GET /analytics/livestreams/overall # Livestream analytics +``` + +### Webhooks +```bash +GET /webhooks # List all +POST /webhooks # Create: {"url": "https://...", "events": ["session.started", "session.ended"]} +PATCH /webhooks/{webhook_id} # Update +DELETE /webhooks/{webhook_id} # Delete +``` + +## Session Lifecycle + +``` +Initialization → Join Intent → [Waitlist?] → Meeting Screen (Stage) → Ended + ↓ Approved + [Rejected → Ended] +``` + +UI Kit handles state transitions automatically. + +## See Also + +- [Configuration](./configuration.md) - Setup and installation +- [Patterns](./patterns.md) - Usage examples +- [README](./README.md) - Overview and quick start diff --git a/plugins/cloudflare/skills/cloudflare/references/realtimekit/configuration.md b/plugins/cloudflare/skills/cloudflare/references/realtimekit/configuration.md new file mode 100644 index 0000000..efbca80 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtimekit/configuration.md @@ -0,0 +1,203 @@ +# RealtimeKit Configuration + +Configuration guide for RealtimeKit setup, client SDKs, and wrangler integration. + +## Installation + +### React +```bash +npm install @cloudflare/realtimekit @cloudflare/realtimekit-react-ui +``` + +### Angular +```bash +npm install @cloudflare/realtimekit @cloudflare/realtimekit-angular-ui +``` + +### Web Components/HTML +```bash +npm install @cloudflare/realtimekit @cloudflare/realtimekit-ui +``` + +## Client SDK Configuration + +### React UI Kit +```tsx +import { RtkMeeting } from '@cloudflare/realtimekit-react-ui'; + {}} /> +``` + +### Angular UI Kit +```typescript +@Component({ template: `` }) +export class AppComponent { authToken = ''; onLeave() {} } +``` + +### Web Components +```html + + + +``` + +### Core SDK Configuration +```typescript +import RealtimeKitClient from '@cloudflare/realtimekit'; + +const meeting = new RealtimeKitClient({ + authToken: '', + video: true, audio: true, autoSwitchAudioDevice: true, + mediaConfiguration: { + video: { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } }, + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + screenshare: { width: { max: 1920 }, height: { max: 1080 }, frameRate: { ideal: 15 } } + } +}); +await meeting.join(); +``` + +## Backend Setup + +### Create App & Credentials + +**Dashboard**: https://dash.cloudflare.com/?to=/:account/realtime/kit + +**API**: +```bash +curl -X POST 'https://api.cloudflare.com/client/v4/accounts//realtime/kit/apps' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' \ + -d '{"name": "My RealtimeKit App"}' +``` + +**Required Permissions**: API token with **Realtime / Realtime Admin** permissions + +### Create Presets + +```bash +curl -X POST 'https://api.cloudflare.com/client/v4/accounts//realtime/kit//presets' \ + -H 'Authorization: Bearer ' \ + -d '{ + "name": "host", + "permissions": { + "canShareAudio": true, + "canShareVideo": true, + "canRecord": true, + "canLivestream": true, + "canStartStopRecording": true + } + }' +``` + +## Wrangler Configuration + +### Basic Configuration +```jsonc +// wrangler.jsonc +{ + "name": "realtimekit-app", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date + "vars": { + "CLOUDFLARE_ACCOUNT_ID": "abc123", + "REALTIMEKIT_APP_ID": "xyz789" + } + // Secrets: wrangler secret put CLOUDFLARE_API_TOKEN +} +``` + +### With Database & Storage +```jsonc +{ + "d1_databases": [{ "binding": "DB", "database_name": "meetings", "database_id": "d1-id" }], + "r2_buckets": [{ "binding": "RECORDINGS", "bucket_name": "recordings" }], + "kv_namespaces": [{ "binding": "SESSIONS", "id": "kv-id" }] +} +``` + +### Multi-Environment +```bash +# Deploy to environments +wrangler deploy --env staging +wrangler deploy --env production +``` + +## TURN Service Configuration + +RealtimeKit can use Cloudflare's TURN service for connectivity through restrictive networks: + +```jsonc +// wrangler.jsonc +{ + "vars": { + "TURN_SERVICE_ID": "your_turn_service_id" + } + // Set secret: wrangler secret put TURN_SERVICE_TOKEN +} +``` + +TURN automatically configured when enabled in account - no client-side changes needed. + +## Theming & Design Tokens + +```typescript +import type { UIConfig } from '@cloudflare/realtimekit'; + +const uiConfig: UIConfig = { + designTokens: { + colors: { + brand: { 500: '#0066ff', 600: '#0052cc' }, + background: { 1000: '#1A1A1A', 900: '#2D2D2D' }, + text: { 1000: '#FFFFFF', 900: '#E0E0E0' } + }, + borderRadius: 'extra-rounded', // 'rounded' | 'extra-rounded' | 'sharp' + theme: 'dark' // 'light' | 'dark' + }, + logo: { url: 'https://example.com/logo.png', altText: 'Company' } +}; + +// Apply to React + {}} /> + +// Or use CSS variables +// :root { --rtk-color-brand-500: #0066ff; --rtk-border-radius: 12px; } +``` + +## Internationalization (i18n) + +### Custom Language Strings +```typescript +import { useLanguage } from '@cloudflare/realtimekit-ui'; + +const customLanguage = { + 'join': 'Entrar', + 'leave': 'Salir', + 'mute': 'Silenciar', + 'unmute': 'Activar audio', + 'turn_on_camera': 'Encender cámara', + 'turn_off_camera': 'Apagar cámara', + 'share_screen': 'Compartir pantalla', + 'stop_sharing': 'Dejar de compartir' +}; + +const t = useLanguage(customLanguage); + +// React usage + {}} /> +``` + +### Supported Locales +Default locales available: `en`, `es`, `fr`, `de`, `pt`, `ja`, `zh` + +```typescript +import { setLocale } from '@cloudflare/realtimekit-ui'; +setLocale('es'); // Switch to Spanish +``` + +## See Also + +- [API](./api.md) - Meeting APIs, REST endpoints +- [Patterns](./patterns.md) - Backend integration examples +- [README](./README.md) - Overview and quick start diff --git a/plugins/cloudflare/skills/cloudflare/references/realtimekit/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/realtimekit/gotchas.md new file mode 100644 index 0000000..c6e7dfd --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtimekit/gotchas.md @@ -0,0 +1,169 @@ +# RealtimeKit Gotchas & Troubleshooting + +## Common Errors + +### "Cannot connect to meeting" + +**Cause:** Auth token invalid/expired, API credentials lack permissions, or network blocks WebRTC +**Solution:** +Verify token validity, check API token has **Realtime / Realtime Admin** permissions, enable TURN service for restrictive networks + +### "No video/audio tracks" + +**Cause:** Browser permissions not granted, video/audio not enabled, device in use, or device unavailable +**Solution:** +Request browser permissions explicitly, verify initialization config, use `meeting.self.getAllDevices()` to debug, close other apps using device + +### "Participant count mismatched" + +**Cause:** `meeting.participants` doesn't include `meeting.self` +**Solution:** Total count = `meeting.participants.joined.size() + 1` + +### "Events not firing" + +**Cause:** Listeners registered after actions, incorrect event name, or wrong namespace +**Solution:** +Register listeners before calling `meeting.join()`, check event names against docs, verify correct namespace + +### "CORS errors in API calls" + +**Cause:** Making REST API calls from client-side +**Solution:** All REST API calls **must** be server-side (Workers, backend). Never expose API tokens to clients. + +### "Preset not applying" + +**Cause:** Preset doesn't exist, name mismatch (case-sensitive), or participant created before preset +**Solution:** +Verify preset exists via Dashboard or API, check exact spelling and case, create preset before adding participants + +### "Token reuse error" + +**Cause:** Reusing participant tokens across sessions +**Solution:** Generate fresh token per session. Use refresh endpoint if token expires during session. + +### "Video quality poor" + +**Cause:** Insufficient bandwidth, resolution/bitrate too high, or CPU overload +**Solution:** +Lower `mediaConfiguration.video` resolution/frameRate, monitor network conditions, reduce participant count or grid size + +### "Echo or audio feedback" + +**Cause:** Multiple devices picking up same audio source +**Solution:** +- Lower `mediaConfiguration.video` resolution/frameRate +- Monitor network conditions +- Reduce participant count or grid size + +### Issue: Echo or audio feedback +**Cause**: Multiple devices picking up same audio source + +**Solutions**: +Enable `echoCancellation: true` in `mediaConfiguration.audio`, use headphones, mute when not speaking + +### "Screen share not working" + +**Cause:** Browser doesn't support screen sharing API, permission denied, or wrong `displaySurface` config +**Solution:** +Use Chrome/Edge/Firefox (Safari limited support), check browser permissions, try different `displaySurface` values ('window', 'monitor', 'browser') + +### "How do I schedule meetings?" + +**Cause:** RealtimeKit has no built-in scheduling system +**Solution:** +Store meeting IDs in your database with timestamps. Generate participant tokens only when user should join. Example: +```typescript +// Store in DB +{ meetingId: 'abc123', scheduledFor: '2026-02-15T10:00:00Z', userId: 'user456' } + +// Generate token when user clicks "Join" near scheduled time +const response = await fetch('/api/join-meeting', { + method: 'POST', + body: JSON.stringify({ meetingId: 'abc123' }) +}); +const { authToken } = await response.json(); +``` + +### "Recording not starting" + +**Cause:** Preset lacks recording permissions, no active session, or API call from client +**Solution:** +Verify preset has `canRecord: true` and `canStartStopRecording: true`, ensure session is active (at least one participant), make recording API calls server-side only + +## Limits + +| Resource | Limit | +|----------|-------| +| Max participants per session | 100 | +| Max concurrent sessions per App | 1000 | +| Max recording duration | 6 hours | +| Max meeting duration | 24 hours | +| Max chat message length | 4000 characters | +| Max preset name length | 64 characters | +| Max meeting title length | 256 characters | +| Max participant name length | 256 characters | +| Token expiration | 24 hours (default) | +| WebRTC ports required | UDP 1024-65535 | + +## Network Requirements + +### Firewall Rules +Allow outbound UDP/TCP to: +- `*.cloudflare.com` ports 443, 80 +- UDP ports 1024-65535 (WebRTC media) + +### TURN Service +Enable for users behind restrictive firewalls/proxies: +```jsonc +// wrangler.jsonc +{ + "vars": { + "TURN_SERVICE_ID": "your_turn_service_id" + } + // Set secret: wrangler secret put TURN_SERVICE_TOKEN +} +``` + +TURN automatically configured in SDK when enabled in account. + +## Debugging Tips + +```typescript +// Check devices +const devices = await meeting.self.getAllDevices(); +meeting.self.on('deviceListUpdate', ({ added, removed, devices }) => console.log('Devices:', { added, removed, devices })); + +// Monitor participants +meeting.participants.joined.on('participantJoined', (p) => console.log(`${p.name} joined:`, { id: p.id, userId: p.userId, audioEnabled: p.audioEnabled, videoEnabled: p.videoEnabled })); + +// Check room state +meeting.self.on('roomJoined', () => console.log('Room:', { meetingId: meeting.meta.meetingId, meetingTitle: meeting.meta.meetingTitle, participantCount: meeting.participants.joined.size() + 1, audioEnabled: meeting.self.audioEnabled, videoEnabled: meeting.self.videoEnabled })); + +// Log all events +['roomJoined', 'audioUpdate', 'videoUpdate', 'screenShareUpdate', 'deviceUpdate', 'deviceListUpdate'].forEach(event => meeting.self.on(event, (data) => console.log(`[self] ${event}:`, data))); +['participantJoined', 'participantLeft'].forEach(event => meeting.participants.joined.on(event, (data) => console.log(`[participants] ${event}:`, data))); +meeting.chat.on('chatUpdate', (data) => console.log('[chat] chatUpdate:', data)); +``` + +## Security & Performance + +### Security: Do NOT +- Expose `CLOUDFLARE_API_TOKEN` in client code, hardcode credentials in frontend +- Reuse participant tokens, store tokens in localStorage without encryption +- Allow client-side meeting creation + +### Security: DO +- Generate tokens server-side only, use HTTPS, implement rate limiting +- Validate user auth before generating tokens, use `custom_participant_id` to map to your user system +- Set appropriate preset permissions per user role, rotate API tokens regularly + +### Performance +- **CPU**: Lower video resolution/frameRate, disable video for audio-only, use `meeting.participants.active` for large meetings, implement virtual scrolling +- **Bandwidth**: Set max resolution in `mediaConfiguration`, disable screenshare audio if unneeded, use audio-only mode, implement adaptive bitrate +- **Memory**: Clean up event listeners on unmount, call `meeting.leave()` when done, don't store large participant arrays + +## In This Reference +- [README.md](README.md) - Overview, core concepts, quick start +- [configuration.md](configuration.md) - SDK config, presets, wrangler setup +- [api.md](api.md) - Client SDK APIs, REST endpoints +- [patterns.md](patterns.md) - Common patterns, React hooks, backend integration diff --git a/plugins/cloudflare/skills/cloudflare/references/realtimekit/patterns.md b/plugins/cloudflare/skills/cloudflare/references/realtimekit/patterns.md new file mode 100644 index 0000000..ac662ef --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/realtimekit/patterns.md @@ -0,0 +1,223 @@ +# RealtimeKit Patterns + +## UI Kit (Minimal Code) + +```tsx +// React +import { RtkMeeting } from '@cloudflare/realtimekit-react-ui'; + console.log('Left')} /> + +// Angular +@Component({ template: `` }) +export class AppComponent { authToken = ''; onLeave(event: unknown) {} } + +// HTML/Web Components + + + +``` + +## UI Components + +RealtimeKit provides 133+ pre-built Stencil.js Web Components with framework wrappers: + +### Layout Components +- `` - Full meeting UI (all-in-one) +- ``, ``, `` - Layout sections +- `` - Chat/participants sidebar +- `` - Adaptive video grid + +### Control Components +- ``, `` - Media controls +- `` - Screen sharing +- `` - Leave meeting +- `` - Device settings + +### Grid Variants +- `` - Active speaker focus +- `` - Audio-only mode +- `` - Paginated layout + +**See full catalog**: https://docs.realtime.cloudflare.com/ui-kit + +## Core SDK Patterns + +### Basic Setup +```typescript +import RealtimeKitClient from '@cloudflare/realtimekit'; + +const meeting = new RealtimeKitClient({ authToken, video: true, audio: true }); +meeting.self.on('roomJoined', () => console.log('Joined:', meeting.meta.meetingTitle)); +meeting.participants.joined.on('participantJoined', (p) => console.log(`${p.name} joined`)); +await meeting.join(); +``` + +### Video Grid & Device Selection +```typescript +// Video grid +function VideoGrid({ meeting }) { + const [participants, setParticipants] = useState([]); + useEffect(() => { + const update = () => setParticipants(meeting.participants.joined.toArray()); + meeting.participants.joined.on('participantJoined', update); + meeting.participants.joined.on('participantLeft', update); + update(); + return () => { meeting.participants.joined.off('participantJoined', update); meeting.participants.joined.off('participantLeft', update); }; + }, [meeting]); + return
+ {participants.map(p => )} +
; +} + +function VideoTile({ participant }) { + const videoRef = useRef(null); + useEffect(() => { + if (videoRef.current && participant.videoTrack) videoRef.current.srcObject = new MediaStream([participant.videoTrack]); + }, [participant.videoTrack]); + return
; +} + +// Device selection +const devices = await meeting.self.getAllDevices(); +const switchCamera = (deviceId: string) => { + const device = devices.find(d => d.deviceId === deviceId); + if (device) await meeting.self.setDevice(device); +}; +``` + +## React Hooks (Official) + +```typescript +import { useRealtimeKitClient, useRealtimeKitSelector } from '@cloudflare/realtimekit-react-ui'; + +function MyComponent() { + const [meeting, initMeeting] = useRealtimeKitClient(); + const audioEnabled = useRealtimeKitSelector(m => m.self.audioEnabled); + const participantCount = useRealtimeKitSelector(m => m.participants.joined.size()); + + useEffect(() => { initMeeting({ authToken: '' }); }, []); + + return
+ + {participantCount} participants +
; +} +``` + +**Benefits:** Automatic re-renders, memoized selectors, type-safe + +## Waitlist Handling + +```typescript +// Monitor waitlist +meeting.participants.waitlisted.on('participantJoined', (participant) => { + console.log(`${participant.name} is waiting`); + // Show admin UI to approve/reject +}); + +// Approve from waitlist (backend only) +await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/realtime/kit/${appId}/meetings/${meetingId}/active-session/waitlist/approve`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}` }, + body: JSON.stringify({ user_ids: [participant.userId] }) + } +); + +// Client receives automatic transition when approved +meeting.self.on('roomJoined', () => console.log('Approved and joined')); +``` + +## Audio-Only Mode + +```typescript +const meeting = new RealtimeKitClient({ + authToken: '', + video: false, // Disable video + audio: true, + mediaConfiguration: { + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + } +}); + +// Use audio grid component +import { RtkAudioGrid } from '@cloudflare/realtimekit-react-ui'; + +``` + +## Addon System + +```typescript +// List available addons +meeting.plugins.all.forEach(plugin => { + console.log(plugin.id, plugin.name, plugin.active); +}); + +// Activate collaborative app +await meeting.plugins.activate('whiteboard-addon-id'); + +// Listen for activations +meeting.plugins.on('pluginActivated', ({ plugin }) => { + console.log(`${plugin.name} activated`); +}); + +// Deactivate +await meeting.plugins.deactivate(); +``` + +## Backend Integration + +### Token Generation (Workers) +```typescript +export interface Env { CLOUDFLARE_API_TOKEN: string; CLOUDFLARE_ACCOUNT_ID: string; REALTIMEKIT_APP_ID: string; } + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + + if (url.pathname === '/api/join-meeting') { + const { meetingId, userName, presetName } = await request.json(); + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/realtime/kit/${env.REALTIMEKIT_APP_ID}/meetings/${meetingId}/participants`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${env.CLOUDFLARE_API_TOKEN}` }, + body: JSON.stringify({ name: userName, preset_name: presetName }) + } + ); + const data = await response.json(); + return Response.json({ authToken: data.result.authToken }); + } + + return new Response('Not found', { status: 404 }); + } +}; +``` + +## Best Practices + +### Security +1. **Never expose API tokens client-side** - Generate participant tokens server-side only +2. **Don't reuse participant tokens** - Generate fresh token per session, use refresh endpoint if expired +3. **Use custom participant IDs** - Map to your user system for cross-session tracking + +### Performance +1. **Event-driven updates** - Listen to events, don't poll. Use `toArray()` only when needed +2. **Media quality constraints** - Set appropriate resolution/bitrate limits based on network conditions +3. **Device management** - Enable `autoSwitchAudioDevice` for better UX, handle device list updates + +### Architecture +1. **Separate Apps for environments** - staging vs production to prevent data mixing +2. **Preset strategy** - Create presets at App level, reuse across meetings +3. **Token management** - Backend generates tokens, frontend receives via authenticated endpoint + +## In This Reference +- [README.md](README.md) - Overview, core concepts, quick start +- [configuration.md](configuration.md) - SDK config, presets, wrangler setup +- [api.md](api.md) - Client SDK APIs, REST endpoints +- [gotchas.md](gotchas.md) - Common issues, troubleshooting, limits diff --git a/plugins/cloudflare/skills/cloudflare/references/sandbox/README.md b/plugins/cloudflare/skills/cloudflare/references/sandbox/README.md new file mode 100644 index 0000000..8550be4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/sandbox/README.md @@ -0,0 +1,96 @@ +# Cloudflare Sandbox SDK + +Secure isolated code execution in containers on Cloudflare's edge. Run untrusted code, manage files, expose services, integrate with AI agents. + +**Use cases**: AI code execution, interactive dev environments, data analysis, CI/CD, code interpreters, multi-tenant execution. + +## Architecture + +- Each sandbox = Durable Object + Container +- Persistent across requests (same ID = same sandbox) +- Isolated filesystem/processes/network +- Configurable sleep/wake for cost optimization + +## Quick Start + +```typescript +import { getSandbox, proxyToSandbox, type Sandbox } from '@cloudflare/sandbox'; +export { Sandbox } from '@cloudflare/sandbox'; + +type Env = { Sandbox: DurableObjectNamespace; }; + +export default { + async fetch(request: Request, env: Env): Promise { + // CRITICAL: proxyToSandbox MUST be called first for preview URLs + const proxyResponse = await proxyToSandbox(request, env); + if (proxyResponse) return proxyResponse; + + const sandbox = getSandbox(env.Sandbox, 'my-sandbox'); + const result = await sandbox.exec('python3 -c "print(2 + 2)"'); + return Response.json({ output: result.stdout }); + } +}; +``` + +**wrangler.jsonc**: +```jsonc +{ + "name": "my-sandbox-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + + "containers": [{ + "class_name": "Sandbox", + "image": "./Dockerfile", + "instance_type": "lite", // lite | standard | heavy + "max_instances": 5 + }], + + "durable_objects": { + "bindings": [{ "class_name": "Sandbox", "name": "Sandbox" }] + }, + + "migrations": [{ + "tag": "v1", + "new_sqlite_classes": ["Sandbox"] + }] +} +``` + +**Dockerfile**: +```dockerfile +FROM docker.io/cloudflare/sandbox:latest +RUN pip3 install --no-cache-dir pandas numpy matplotlib +EXPOSE 8080 3000 # Required for wrangler dev +``` + +## Core APIs + +- `getSandbox(namespace, id, options?)` → Get/create sandbox +- `sandbox.exec(command, options?)` → Execute command +- `sandbox.readFile(path)` / `writeFile(path, content)` → File ops +- `sandbox.startProcess(command, options)` → Background process +- `sandbox.exposePort(port, options)` → Get preview URL +- `sandbox.createSession(options)` → Isolated session +- `sandbox.wsConnect(request, port)` → WebSocket proxy +- `sandbox.destroy()` → Terminate container +- `sandbox.mountBucket(bucket, path, options)` → Mount S3 storage + +## Critical Rules + +- ALWAYS call `proxyToSandbox()` first +- Same ID = reuse sandbox +- Use `/workspace` for persistent files +- `normalizeId: true` for preview URLs +- Retry on `CONTAINER_NOT_READY` + +## In This Reference +- [configuration.md](./configuration.md) - Config, CLI, environment setup +- [api.md](./api.md) - Programmatic API, testing patterns +- [patterns.md](./patterns.md) - Common workflows, CI/CD integration +- [gotchas.md](./gotchas.md) - Issues, limits, best practices + +## See Also +- [durable-objects](../durable-objects/) - Sandbox runs on DO infrastructure +- [containers](../containers/) - Container runtime fundamentals +- [workers](../workers/) - Entry point for sandbox requests diff --git a/plugins/cloudflare/skills/cloudflare/references/sandbox/api.md b/plugins/cloudflare/skills/cloudflare/references/sandbox/api.md new file mode 100644 index 0000000..3eb2fa5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/sandbox/api.md @@ -0,0 +1,198 @@ +# API Reference + +## Command Execution + +```typescript +// Basic +const result = await sandbox.exec('python3 script.py'); +// Returns: { stdout, stderr, exitCode, success, duration } + +// With options +await sandbox.exec('python3 test.py', { + cwd: '/workspace/project', + env: { API_KEY: 'secret' }, + stream: true, + onOutput: (stream, data) => console.log(data) +}); +``` + +## File Operations + +```typescript +// Read/Write +const { content } = await sandbox.readFile('/workspace/data.txt'); +await sandbox.writeFile('/workspace/file.txt', 'content'); // Auto-creates dirs + +// List/Delete +const files = await sandbox.listFiles('/workspace'); +await sandbox.deleteFile('/workspace/temp.txt'); +await sandbox.deleteFile('/workspace/dir', { recursive: true }); + +// Utils +await sandbox.mkdir('/workspace/dir', { recursive: true }); +await sandbox.pathExists('/workspace/file.txt'); +``` + +## Background Processes + +```typescript +// Start +const process = await sandbox.startProcess('python3 -m http.server 8080', { + processId: 'web-server', + cwd: '/workspace/public', + env: { PORT: '8080' } +}); +// Returns: { id, pid, command } + +// Wait for readiness +await process.waitForPort(8080); // Wait for port to listen +await process.waitForLog(/Server running/); // Wait for log pattern +await process.waitForExit(); // Wait for completion + +// Management +const processes = await sandbox.listProcesses(); +const info = await sandbox.getProcess('web-server'); +await sandbox.stopProcess('web-server'); +const logs = await sandbox.getProcessLogs('web-server'); +``` + +## Port Exposure + +```typescript +// Expose port +const { url } = await sandbox.exposePort(8080, { + name: 'web-app', + hostname: request.hostname +}); + +// Management +await sandbox.isPortExposed(8080); +await sandbox.getExposedPorts(request.hostname); +await sandbox.unexposePort(8080); +``` + +## Sessions (Isolated Contexts) + +Each session maintains own shell state, env vars, cwd, process namespace. + +```typescript +// Create with context +const session = await sandbox.createSession({ + id: 'user-123', + cwd: '/workspace/user123', + env: { USER_ID: '123' } +}); + +// Use (full sandbox API) +await session.exec('echo $USER_ID'); +await session.writeFile('config.txt', 'data'); + +// Manage +await sandbox.getSession('user-123'); +await sandbox.deleteSession('user-123'); +``` + +## Code Interpreter + +```typescript +// Create context with variables +const ctx = await sandbox.createCodeContext({ + language: 'python', + variables: { + data: [1, 2, 3, 4, 5], + config: { verbose: true } + } +}); + +// Execute code with rich outputs +const result = await ctx.runCode(` +import matplotlib.pyplot as plt +plt.plot(data, [x**2 for x in data]) +plt.savefig('plot.png') +print(f"Processed {len(data)} points") +`); +// Returns: { outputs: [{ type: 'text'|'image'|'html', content }], error } + +// Context persists variables across runs +const result2 = await ctx.runCode('print(data[0])'); // Still has 'data' +``` + +## WebSocket Connections + +```typescript +// Proxy WebSocket to sandbox service +export default { + async fetch(request: Request, env: Env): Promise { + const proxyResponse = await proxyToSandbox(request, env); + if (proxyResponse) return proxyResponse; + + if (request.headers.get('Upgrade')?.toLowerCase() === 'websocket') { + const sandbox = getSandbox(env.Sandbox, 'realtime'); + return await sandbox.wsConnect(request, 8080); + } + + return new Response('Not a WebSocket request', { status: 400 }); + } +}; +``` + +## Bucket Mounting (S3 Storage) + +```typescript +// Mount R2 bucket (production only, not wrangler dev) +await sandbox.mountBucket(env.DATA_BUCKET, '/data', { + readOnly: false +}); + +// Access files in mounted bucket +await sandbox.exec('ls /data'); +await sandbox.writeFile('/data/output.txt', 'result'); + +// Unmount +await sandbox.unmountBucket('/data'); +``` + +**Note**: Bucket mounting only works in production. Mounted buckets are sandbox-scoped (visible to all sessions in that sandbox). + +## Lifecycle Management + +```typescript +// Terminate container immediately +await sandbox.destroy(); + +// REQUIRED when using keepAlive: true +const sandbox = getSandbox(env.Sandbox, 'temp', { keepAlive: true }); +try { + await sandbox.writeFile('/tmp/code.py', code); + const result = await sandbox.exec('python /tmp/code.py'); + return result.stdout; +} finally { + await sandbox.destroy(); // Free resources +} +``` + +Deletes: files, processes, sessions, network connections, exposed ports. + +## Error Handling + +```typescript +// Command errors +const result = await sandbox.exec('python3 invalid.py'); +if (!result.success) { + console.error('Exit code:', result.exitCode); + console.error('Stderr:', result.stderr); +} + +// SDK errors +try { + await sandbox.readFile('/nonexistent'); +} catch (error) { + if (error.code === 'FILE_NOT_FOUND') { /* ... */ } + else if (error.code === 'CONTAINER_NOT_READY') { /* retry */ } + else if (error.code === 'TIMEOUT') { /* ... */ } +} + +// Retry pattern (see gotchas.md for full implementation) +``` + + diff --git a/plugins/cloudflare/skills/cloudflare/references/sandbox/configuration.md b/plugins/cloudflare/skills/cloudflare/references/sandbox/configuration.md new file mode 100644 index 0000000..32a3bd9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/sandbox/configuration.md @@ -0,0 +1,143 @@ +# Configuration + +## getSandbox Options + +```typescript +const sandbox = getSandbox(env.Sandbox, 'sandbox-id', { + normalizeId: true, // lowercase ID (required for preview URLs) + sleepAfter: '10m', // sleep after inactivity: '5m', '1h', '2d' (default: '10m') + keepAlive: false, // false = auto-timeout, true = never sleep + + containerTimeouts: { + instanceGetTimeoutMS: 30000, // 30s for provisioning (default: 30000) + portReadyTimeoutMS: 90000 // 90s for container startup (default: 90000) + } +}); +``` + +**Sleep Config**: +- `sleepAfter`: Duration string (e.g., '5m', '10m', '1h') - default: '10m' +- `keepAlive: false`: Auto-sleep (default, cost-optimized) +- `keepAlive: true`: Never sleep (higher cost, requires explicit `destroy()`) +- Sleeping sandboxes wake automatically (cold start) + +## Instance Types + +wrangler.jsonc `instance_type`: +- `lite`: 256MB RAM, 0.5 vCPU (default) +- `standard`: 512MB RAM, 1 vCPU +- `heavy`: 1GB RAM, 2 vCPU + +## Dockerfile Patterns + +**Basic**: +```dockerfile +FROM docker.io/cloudflare/sandbox:latest +RUN pip3 install --no-cache-dir pandas numpy +EXPOSE 8080 # Required for wrangler dev +``` + +**Scientific**: +```dockerfile +FROM docker.io/cloudflare/sandbox:latest +RUN pip3 install --no-cache-dir \ + jupyter-server ipykernel matplotlib \ + pandas seaborn plotly scipy scikit-learn +``` + +**Node.js**: +```dockerfile +FROM docker.io/cloudflare/sandbox:latest +RUN npm install -g typescript ts-node +``` + +**CRITICAL**: `EXPOSE` required for `wrangler dev` port access. Production auto-exposes all ports. + +## CLI Commands + +```bash +# Dev +wrangler dev # Start local dev server +wrangler deploy # Deploy to production +wrangler tail # Monitor logs +wrangler containers list # Check container status +wrangler secret put KEY # Set secret +``` + +## Environment & Secrets + +**wrangler.jsonc**: +```jsonc +{ + "vars": { + "ENVIRONMENT": "production", + "API_URL": "https://api.example.com" + }, + "r2_buckets": [{ + "binding": "DATA_BUCKET", + "bucket_name": "my-data-bucket" + }] +} +``` + +**Usage**: +```typescript +const token = env.GITHUB_TOKEN; // From wrangler secret +await sandbox.exec('git clone ...', { + env: { GIT_TOKEN: token } +}); +``` + +## Preview URL Setup + +**Prerequisites**: +- Custom domain with wildcard DNS: `*.yourdomain.com → worker.yourdomain.com` +- `.workers.dev` domains NOT supported +- `normalizeId: true` in getSandbox +- `proxyToSandbox()` called first in fetch handler + +## Cron Triggers (Pre-warming) + +```jsonc +{ + "triggers": { + "crons": ["*/5 * * * *"] // Every 5 minutes + } +} +``` + +```typescript +export default { + async scheduled(event: ScheduledEvent, env: Env) { + const sandbox = getSandbox(env.Sandbox, 'main'); + await sandbox.exec('echo "keepalive"'); // Wake sandbox + } +}; +``` + +## Logging Configuration + +**wrangler.jsonc**: +```jsonc +{ + "vars": { + "SANDBOX_LOG_LEVEL": "debug", // debug | info | warn | error (default: info) + "SANDBOX_LOG_FORMAT": "pretty" // json | pretty (default: json) + } +} +``` + +**Dev**: `debug` + `pretty`. **Production**: `info`/`warn` + `json`. + +## Timeout Environment Overrides + +Override default timeouts via environment variables: + +```jsonc +{ + "vars": { + "SANDBOX_INSTANCE_TIMEOUT_MS": "60000", // Override instanceGetTimeoutMS + "SANDBOX_PORT_TIMEOUT_MS": "120000" // Override portReadyTimeoutMS + } +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/sandbox/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/sandbox/gotchas.md new file mode 100644 index 0000000..856c503 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/sandbox/gotchas.md @@ -0,0 +1,194 @@ +# Gotchas & Best Practices + +## Common Errors + +### "Container running indefinitely" + +**Cause:** `keepAlive: true` without calling `destroy()` +**Solution:** Always call `destroy()` when done with keepAlive containers + +```typescript +const sandbox = getSandbox(env.Sandbox, 'temp', { keepAlive: true }); +try { + const result = await sandbox.exec('python script.py'); + return result.stdout; +} finally { + await sandbox.destroy(); // REQUIRED to free resources +} +``` + +### "CONTAINER_NOT_READY" + +**Cause:** Container still provisioning (first request or after sleep) +**Solution:** Retry after 2-3s + +```typescript +async function execWithRetry(sandbox, cmd) { + for (let i = 0; i < 3; i++) { + try { + return await sandbox.exec(cmd); + } catch (e) { + if (e.code === 'CONTAINER_NOT_READY') { + await new Promise(r => setTimeout(r, 2000)); + continue; + } + throw e; + } + } +} +``` + +### "Connection refused: container port not found" + +**Cause:** Missing `EXPOSE` directive in Dockerfile +**Solution:** Add `EXPOSE ` to Dockerfile (only needed for `wrangler dev`, production auto-exposes) + +### "Preview URLs not working" + +**Cause:** Custom domain not configured, wildcard DNS missing, `normalizeId` not set, or `proxyToSandbox()` not called +**Solution:** Check: +1. Custom domain configured? (not `.workers.dev`) +2. Wildcard DNS set up? (`*.domain.com → worker.domain.com`) +3. `normalizeId: true` in getSandbox? +4. `proxyToSandbox()` called first in fetch? + +### "Slow first request" + +**Cause:** Cold start (container provisioning) +**Solution:** +- Use `sleepAfter` instead of creating new sandboxes +- Pre-warm with cron triggers +- Set `keepAlive: true` for critical sandboxes + +### "File not persisting" + +**Cause:** Files in `/tmp` or other ephemeral paths +**Solution:** Use `/workspace` for persistent files + +### "Bucket mounting doesn't work locally" + +**Cause:** Bucket mounting requires FUSE, not available in `wrangler dev` +**Solution:** Test bucket mounting in production only. Use mock data locally. + +### "Different normalizeId = different sandbox" + +**Cause:** Changing `normalizeId` option changes Durable Object ID +**Solution:** Set `normalizeId` consistently. `normalizeId: true` lowercases the ID. + +```typescript +// These create DIFFERENT sandboxes: +getSandbox(env.Sandbox, 'MyApp'); // DO ID: hash('MyApp') +getSandbox(env.Sandbox, 'MyApp', { normalizeId: true }); // DO ID: hash('myapp') +``` + +### "Code context variables disappeared" + +**Cause:** Container restart clears code context state +**Solution:** Code contexts are ephemeral. Recreate context after container sleep/wake. + +## Performance Optimization + +### Sandbox ID Strategy + +```typescript +// ❌ BAD: New sandbox every time (slow) +const sandbox = getSandbox(env.Sandbox, `user-${Date.now()}`); + +// ✅ GOOD: Reuse per user +const sandbox = getSandbox(env.Sandbox, `user-${userId}`); +``` + +### Sleep & Traffic Config + +```typescript +// Cost-optimized +getSandbox(env.Sandbox, 'id', { sleepAfter: '30m', keepAlive: false }); + +// Always-on (requires destroy()) +getSandbox(env.Sandbox, 'id', { keepAlive: true }); +``` + +```jsonc +// High traffic: increase max_instances +{ "containers": [{ "class_name": "Sandbox", "max_instances": 50 }] } +``` + +## Security Best Practices + +### Sandbox Isolation +- Each sandbox = isolated container (filesystem, network, processes) +- Use unique sandbox IDs per tenant for multi-tenant apps +- Sandboxes cannot communicate directly + +### Input Validation + +```typescript +// ❌ DANGEROUS: Command injection +const result = await sandbox.exec(`python3 -c "${userCode}"`); + +// ✅ SAFE: Write to file, execute file +await sandbox.writeFile('/workspace/user_code.py', userCode); +const result = await sandbox.exec('python3 /workspace/user_code.py'); +``` + +### Resource Limits + +```typescript +// Timeout long-running commands +const result = await sandbox.exec('python3 script.py', { + timeout: 30000 // 30 seconds +}); +``` + +### Secrets Management + +```typescript +// ❌ NEVER hardcode secrets +const token = 'ghp_abc123'; + +// ✅ Use environment secrets +const token = env.GITHUB_TOKEN; + +// Pass to sandbox via exec env +const result = await sandbox.exec('git clone ...', { + env: { GIT_TOKEN: token } +}); +``` + +### Preview URL Security +Preview URLs include auto-generated tokens: +``` +https://8080-sandbox-abc123def456.yourdomain.com +``` +Token changes on each expose operation, preventing unauthorized access. + +## Limits + +| Resource | Lite | Standard | Heavy | +|----------|------|----------|-------| +| RAM | 256MB | 512MB | 1GB | +| vCPU | 0.5 | 1 | 2 | + +| Operation | Default Timeout | Override | +|-----------|----------------|----------| +| Container provisioning | 30s | `SANDBOX_INSTANCE_TIMEOUT_MS` | +| Port readiness | 90s | `SANDBOX_PORT_TIMEOUT_MS` | +| exec() | 120s | `timeout` option | +| sleepAfter | 10m | `sleepAfter` option | + +**Performance**: +- **First deploy**: 2-3 min for container build +- **Cold start**: 2-3s when waking from sleep +- **Bucket mounting**: Production only (FUSE not in dev) + +## Production Guide + +See: https://developers.cloudflare.com/sandbox/guides/production-deployment/ + +## Resources + +- [Official Docs](https://developers.cloudflare.com/sandbox/) +- [API Reference](https://developers.cloudflare.com/sandbox/api/) +- [Examples](https://github.com/cloudflare/sandbox-sdk/tree/main/examples) +- [npm Package](https://www.npmjs.com/package/@cloudflare/sandbox) +- [Discord Support](https://discord.cloudflare.com) diff --git a/plugins/cloudflare/skills/cloudflare/references/sandbox/patterns.md b/plugins/cloudflare/skills/cloudflare/references/sandbox/patterns.md new file mode 100644 index 0000000..adeb0a0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/sandbox/patterns.md @@ -0,0 +1,201 @@ +# Common Patterns + +## AI Code Execution with Code Context + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const { code, variables } = await request.json(); + const sandbox = getSandbox(env.Sandbox, 'ai-agent'); + + // Create context with persistent variables + const ctx = await sandbox.createCodeContext({ + language: 'python', + variables: variables || {} + }); + + // Execute with rich outputs (text, images, HTML) + const result = await ctx.runCode(code); + + return Response.json({ + outputs: result.outputs, // [{ type: 'text'|'image'|'html', content }] + error: result.error, + success: !result.error + }); + } +}; +``` + +## Interactive Dev Environment + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const proxyResponse = await proxyToSandbox(request, env); + if (proxyResponse) return proxyResponse; + + const sandbox = getSandbox(env.Sandbox, 'ide', { normalizeId: true }); + + if (request.url.endsWith('/start')) { + await sandbox.exec('curl -fsSL https://code-server.dev/install.sh | sh'); + await sandbox.startProcess('code-server --bind-addr 0.0.0.0:8080', { + processId: 'vscode' + }); + + const exposed = await sandbox.exposePort(8080); + return Response.json({ url: exposed.url }); + } + + return new Response('Try /start'); + } +}; +``` + +## WebSocket Real-Time Service + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const proxyResponse = await proxyToSandbox(request, env); + if (proxyResponse) return proxyResponse; + + if (request.headers.get('Upgrade')?.toLowerCase() === 'websocket') { + const sandbox = getSandbox(env.Sandbox, 'realtime-service'); + return await sandbox.wsConnect(request, 8080); + } + + // Non-WebSocket: expose preview URL + const sandbox = getSandbox(env.Sandbox, 'realtime-service'); + const { url } = await sandbox.exposePort(8080, { + hostname: new URL(request.url).hostname + }); + return Response.json({ wsUrl: url.replace('https', 'wss') }); + } +}; +``` + +**Dockerfile**: +```dockerfile +FROM docker.io/cloudflare/sandbox:latest +RUN npm install -g ws +EXPOSE 8080 +``` + +## Process Readiness Pattern + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const sandbox = getSandbox(env.Sandbox, 'app-server'); + + // Start server + const process = await sandbox.startProcess( + 'node server.js', + { processId: 'server' } + ); + + // Wait for server to be ready + await process.waitForPort(8080); // Wait for port listening + + // Now safe to expose + const { url } = await sandbox.exposePort(8080); + return Response.json({ url }); + } +}; +``` + +## Persistent Data with Bucket Mounting + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const sandbox = getSandbox(env.Sandbox, 'data-processor'); + + // Mount R2 bucket (production only) + await sandbox.mountBucket(env.DATA_BUCKET, '/data', { + readOnly: false + }); + + // Process files in bucket + const result = await sandbox.exec('python3 /workspace/process.py', { + env: { DATA_DIR: '/data/input' } + }); + + // Results written to /data/output are persisted in R2 + return Response.json({ success: result.success }); + } +}; +``` + +## CI/CD Pipeline + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const { repo, branch } = await request.json(); + const sandbox = getSandbox(env.Sandbox, `ci-${repo}-${Date.now()}`); + + await sandbox.exec(`git clone -b ${branch} ${repo} /workspace/repo`); + + const install = await sandbox.exec('npm install', { + cwd: '/workspace/repo', + stream: true, + onOutput: (stream, data) => console.log(data) + }); + + if (!install.success) { + return Response.json({ success: false, error: 'Install failed' }); + } + + const test = await sandbox.exec('npm test', { cwd: '/workspace/repo' }); + + return Response.json({ + success: test.success, + output: test.stdout, + exitCode: test.exitCode + }); + } +}; +``` + + + + + +## Multi-Tenant Pattern + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const userId = request.headers.get('X-User-ID'); + const sandbox = getSandbox(env.Sandbox, 'multi-tenant'); + + // Each user gets isolated session + let session; + try { + session = await sandbox.getSession(userId); + } catch { + session = await sandbox.createSession({ + id: userId, + cwd: `/workspace/users/${userId}`, + env: { USER_ID: userId } + }); + } + + const code = await request.text(); + const result = await session.exec(`python3 -c "${code}"`); + + return Response.json({ output: result.stdout }); + } +}; +``` + +## Git Operations + +```typescript +// Clone repo +await sandbox.exec('git clone https://github.com/user/repo.git /workspace/repo'); + +// Authenticated (use env secrets) +await sandbox.exec(`git clone https://${env.GITHUB_TOKEN}@github.com/user/repo.git`); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/secrets-store/README.md b/plugins/cloudflare/skills/cloudflare/references/secrets-store/README.md new file mode 100644 index 0000000..dc709e9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/secrets-store/README.md @@ -0,0 +1,74 @@ +# Cloudflare Secrets Store + +Account-level encrypted secret management for Workers and AI Gateway. + +## Overview + +**Secrets Store**: Centralized, account-level secrets, reusable across Workers +**Worker Secrets**: Per-Worker secrets (`wrangler secret put`) + +### Architecture + +- **Store**: Container (1/account in beta) +- **Secret**: String ≤1024 bytes +- **Scopes**: Permission boundaries controlling access + - `workers`: For Workers runtime access + - `ai-gateway`: For AI Gateway access + - Secrets must have correct scope for binding to work +- **Bindings**: Connect secrets via `env` object + +**Regional Availability**: Global except China Network (unavailable) + +### Access Control + +- **Super Admin**: Full access +- **Admin**: Create/edit/delete secrets, view metadata +- **Deployer**: View metadata + bindings +- **Reporter**: View metadata only + +API Token permissions: `Account Secrets Store Edit/Read` + +### Limits (Beta) + +- 100 secrets/account +- 1 store/account +- 1024 bytes max/secret +- Production secrets count toward limit + +## When to Use + +**Use Secrets Store when:** +- Multiple Workers share same credential +- Centralized management needed +- Compliance requires audit trail +- Team collaboration on secrets + +**Use Worker Secrets when:** +- Secret unique to one Worker +- Simple single-Worker project +- No cross-Worker sharing needed + +## In This Reference + +### Reading Order by Task + +| Task | Start Here | Then Read | +|------|------------|-----------| +| Quick overview | README.md | - | +| First-time setup | README.md → configuration.md | api.md | +| Add secret to Worker | configuration.md | api.md | +| Implement access pattern | api.md | patterns.md | +| Debug errors | gotchas.md | api.md | +| Secret rotation | patterns.md | configuration.md | +| Best practices | gotchas.md | patterns.md | + +### Files + +- [configuration.md](./configuration.md) - Wrangler commands, binding config +- [api.md](./api.md) - Binding API, get/put/delete operations +- [patterns.md](./patterns.md) - Rotation, encryption, access control +- [gotchas.md](./gotchas.md) - Security issues, limits, best practices + +## See Also +- [workers](../workers/) - Worker bindings integration +- [wrangler](../wrangler/) - CLI secret management commands diff --git a/plugins/cloudflare/skills/cloudflare/references/secrets-store/api.md b/plugins/cloudflare/skills/cloudflare/references/secrets-store/api.md new file mode 100644 index 0000000..2e4e6e2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/secrets-store/api.md @@ -0,0 +1,200 @@ +# API Reference + +## Binding API + +### Basic Access + +**CRITICAL**: Async `.get()` required - secrets NOT directly available. + +**`.get()` throws on error** - does NOT return null. Always use try/catch. + +```typescript +interface Env { + API_KEY: { get(): Promise }; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const apiKey = await env.API_KEY.get(); + return fetch("https://api.example.com", { + headers: { "Authorization": `Bearer ${apiKey}` } + }); + } +} +``` + +### Error Handling + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + try { + const apiKey = await env.API_KEY.get(); + return fetch("https://api.example.com", { + headers: { "Authorization": `Bearer ${apiKey}` } + }); + } catch (error) { + console.error("Secret access failed:", error); + return new Response("Configuration error", { status: 500 }); + } + } +} +``` + +### Multiple Secrets & Patterns + +```typescript +// Parallel fetch +const [stripeKey, sendgridKey] = await Promise.all([ + env.STRIPE_KEY.get(), + env.SENDGRID_KEY.get() +]); + +// ❌ Missing .get() +const key = env.API_KEY; + +// ❌ Module-level cache +const CACHED_KEY = await env.API_KEY.get(); // Fails + +// ✅ Request-scope cache +const key = await env.API_KEY.get(); // OK - reuse within request +``` + +## REST API + +Base: `https://api.cloudflare.com/client/v4` + +### Auth + +```bash +curl -H "Authorization: Bearer $CF_TOKEN" \ + https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/secrets_store/stores +``` + +### Store Operations + +```bash +# List +GET /accounts/{account_id}/secrets_store/stores + +# Create +POST /accounts/{account_id}/secrets_store/stores +{"name": "my-store"} + +# Delete +DELETE /accounts/{account_id}/secrets_store/stores/{store_id} +``` + +### Secret Operations + +```bash +# List +GET /accounts/{account_id}/secrets_store/stores/{store_id}/secrets + +# Create (single) +POST /accounts/{account_id}/secrets_store/stores/{store_id}/secrets +{ + "name": "my_secret", + "value": "secret_value", + "scopes": ["workers"], + "comment": "Optional" +} + +# Create (batch) +POST /accounts/{account_id}/secrets_store/stores/{store_id}/secrets +[ + {"name": "secret_one", "value": "val1", "scopes": ["workers"]}, + {"name": "secret_two", "value": "val2", "scopes": ["workers", "ai-gateway"]} +] + +# Get metadata +GET /accounts/{account_id}/secrets_store/stores/{store_id}/secrets/{secret_id} + +# Update +PATCH /accounts/{account_id}/secrets_store/stores/{store_id}/secrets/{secret_id} +{"value": "new_value", "comment": "Updated"} + +# Delete (single) +DELETE /accounts/{account_id}/secrets_store/stores/{store_id}/secrets/{secret_id} + +# Delete (batch) +DELETE /accounts/{account_id}/secrets_store/stores/{store_id}/secrets +{"secret_ids": ["id-1", "id-2"]} + +# Duplicate +POST /accounts/{account_id}/secrets_store/stores/{store_id}/secrets/{secret_id}/duplicate +{"name": "new_name"} + +# Quota +GET /accounts/{account_id}/secrets_store/quota +``` + +### Responses + +Success: +```json +{ + "success": true, + "result": { + "id": "secret-id-123", + "name": "my_secret", + "created": "2025-01-11T12:00:00Z", + "scopes": ["workers"] + } +} +``` + +Error: +```json +{ + "success": false, + "errors": [{"code": 10000, "message": "Name exists"}] +} +``` + +## TypeScript Helpers + +Official types available via `@cloudflare/workers-types`: + +```typescript +import type { SecretsStoreSecret } from "@cloudflare/workers-types"; + +interface Env { + STRIPE_API_KEY: SecretsStoreSecret; + DATABASE_URL: SecretsStoreSecret; + WORKER_SECRET: string; // Regular Worker secret (direct access) +} +``` + +Custom helper type: + +```typescript +interface SecretsStoreBinding { + get(): Promise; +} + +// Fallback helper +async function getSecretWithFallback( + primary: SecretsStoreBinding, + fallback?: SecretsStoreBinding +): Promise { + try { + return await primary.get(); + } catch (error) { + if (fallback) return await fallback.get(); + throw error; + } +} + +// Batch helper +async function getAllSecrets( + secrets: Record +): Promise> { + const entries = await Promise.all( + Object.entries(secrets).map(async ([k, v]) => [k, await v.get()]) + ); + return Object.fromEntries(entries); +} +``` + +See: [configuration.md](./configuration.md), [patterns.md](./patterns.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/secrets-store/configuration.md b/plugins/cloudflare/skills/cloudflare/references/secrets-store/configuration.md new file mode 100644 index 0000000..a1e2eee --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/secrets-store/configuration.md @@ -0,0 +1,185 @@ +# Configuration + +## Wrangler Config + +### Basic Binding + +**wrangler.jsonc**: + +```jsonc +{ + "secrets_store_secrets": [ + { + "binding": "API_KEY", + "store_id": "abc123", + "secret_name": "stripe_api_key" + } + ] +} +``` + +**wrangler.toml** (alternative): + +```toml +[[secrets_store_secrets]] +binding = "API_KEY" +store_id = "abc123" +secret_name = "stripe_api_key" +``` + +Fields: +- `binding`: Variable name for `env` access +- `store_id`: From `wrangler secrets-store store list` +- `secret_name`: Identifier (no spaces) + +### Environment-Specific + +**wrangler.jsonc**: + +```jsonc +{ + "env": { + "production": { + "secrets_store_secrets": [ + { + "binding": "API_KEY", + "store_id": "prod-store", + "secret_name": "prod_api_key" + } + ] + }, + "staging": { + "secrets_store_secrets": [ + { + "binding": "API_KEY", + "store_id": "staging-store", + "secret_name": "staging_api_key" + } + ] + } + } +} +``` + +**wrangler.toml** (alternative): + +```toml +[env.production] +[[env.production.secrets_store_secrets]] +binding = "API_KEY" +store_id = "prod-store" +secret_name = "prod_api_key" + +[env.staging] +[[env.staging.secrets_store_secrets]] +binding = "API_KEY" +store_id = "staging-store" +secret_name = "staging_api_key" +``` + +## Wrangler Commands + +### Store Management + +```bash +wrangler secrets-store store list +wrangler secrets-store store create my-store --remote +wrangler secrets-store store delete --remote +``` + +### Secret Management (Production) + +```bash +# Create (interactive) +wrangler secrets-store secret create \ + --name MY_SECRET --scopes workers --remote + +# Create (piped) +cat secret.txt | wrangler secrets-store secret create \ + --name MY_SECRET --scopes workers --remote + +# List/get/update/delete +wrangler secrets-store secret list --remote +wrangler secrets-store secret get --name MY_SECRET --remote +wrangler secrets-store secret update --name MY_SECRET --new-value "val" --remote +wrangler secrets-store secret delete --name MY_SECRET --remote + +# Duplicate +wrangler secrets-store secret duplicate \ + --name ORIG --new-name COPY --remote +``` + +### Local Development + +**CRITICAL**: Production secrets (`--remote`) NOT accessible in local dev. + +```bash +# Create local-only (no --remote) +wrangler secrets-store secret create --name DEV_KEY --scopes workers + +wrangler dev # Uses local secrets +wrangler deploy # Uses production secrets +``` + +Best practice: Separate names for local/prod: + +```jsonc +{ + "env": { + "development": { + "secrets_store_secrets": [ + { "binding": "API_KEY", "store_id": "store", "secret_name": "dev_api_key" } + ] + }, + "production": { + "secrets_store_secrets": [ + { "binding": "API_KEY", "store_id": "store", "secret_name": "prod_api_key" } + ] + } + } +} +``` + +## Dashboard + +### Creating Secrets + +1. **Secrets Store** → **Create secret** +2. Fill: Name (no spaces), Value, Scope (`Workers`), Comment +3. **Save** (value hidden after) + +### Adding Bindings + +**Method 1**: Worker → Settings → Bindings → Add → Secrets Store +**Method 2**: Create secret directly from Worker settings dropdown + +Deploy options: +- **Deploy**: Immediate 100% +- **Save version**: Gradual rollout + +## CI/CD + +### GitHub Actions + +```yaml +- name: Create secret + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CF_TOKEN }} + run: | + echo "${{ secrets.API_KEY }}" | \ + npx wrangler secrets-store secret create $STORE_ID \ + --name API_KEY --scopes workers --remote + +- name: Deploy + run: npx wrangler deploy +``` + +### GitLab CI + +```yaml +script: + - echo "$API_KEY_VALUE" | npx wrangler secrets-store secret create $STORE_ID --name API_KEY --scopes workers --remote + - npx wrangler deploy +``` + +See: [api.md](./api.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/secrets-store/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/secrets-store/gotchas.md new file mode 100644 index 0000000..08218b4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/secrets-store/gotchas.md @@ -0,0 +1,97 @@ +# Gotchas + +## Common Errors + +### ".get() Throws on Error" + +**Cause:** Assuming `.get()` returns null on failure instead of throwing +**Solution:** Always wrap `.get()` calls in try/catch blocks to handle errors gracefully + +```typescript +try { + const key = await env.API_KEY.get(); +} catch (error) { + return new Response("Configuration error", { status: 500 }); +} +``` + +### "Logging Secret Values" + +**Cause:** Accidentally logging secret values in console or error messages +**Solution:** Only log metadata (e.g., "Retrieved API_KEY") never the actual secret value + +### "Module-Level Secret Access" + +**Cause:** Attempting to access secrets during module initialization before env is available +**Solution:** Cache secrets in request scope only, not at module level + +### "Secret not found in store" + +**Cause:** Secret name doesn't exist, case mismatch, missing workers scope, or incorrect store_id +**Solution:** Verify secret exists with `wrangler secrets-store secret list --remote`, check name matches exactly (case-sensitive), ensure secret has `workers` scope, and verify correct store_id + +### "Scope Mismatch" + +**Cause:** Secret exists but missing `workers` scope (only has `ai-gateway` scope) +**Solution:** Update secret scopes: `wrangler secrets-store secret update --name SECRET --scopes workers --remote` or add via Dashboard + +### "JSON Parsing Failure" + +**Cause:** Storing invalid JSON in secret, then failing to parse during runtime +**Solution:** Validate JSON before storing: + +```bash +# Validate before storing +echo '{"key":"value"}' | jq . && \ + echo '{"key":"value"}' | wrangler secrets-store secret create \ + --name CONFIG --scopes workers --remote +``` + +Runtime parsing with error handling: + +```typescript +try { + const configStr = await env.CONFIG.get(); + const config = JSON.parse(configStr); +} catch (error) { + console.error("Invalid config JSON:", error); + return new Response("Invalid configuration", { status: 500 }); +} +``` + +### "Cannot access secret in local dev" + +**Cause:** Attempting to access production secrets in local development environment +**Solution:** Create local-only secrets (without `--remote` flag) for development: `wrangler secrets-store secret create --name API_KEY --scopes workers` + +### "Property 'get' does not exist" + +**Cause:** Missing TypeScript type definition for secret binding +**Solution:** Define interface with get method: `interface Env { API_KEY: { get(): Promise }; }` + +### "Binding already exists" + +**Cause:** Duplicate binding in dashboard or conflict between wrangler.jsonc and dashboard +**Solution:** Remove duplicate from dashboard Settings → Bindings, check for conflicts, or delete old Worker secret with `wrangler secret delete API_KEY` + +### "Account secret quota exceeded" + +**Cause:** Account has reached 100 secret limit (beta) +**Solution:** Check quota with `wrangler secrets-store quota --remote`, delete unused secrets, consolidate duplicates, or contact Cloudflare for increase + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Max secrets per account | 100 | Beta limit | +| Max stores per account | 1 | Beta limit | +| Max secret size | 1024 bytes | Per secret | +| Local secrets | Don't count toward limit | Only production secrets count | +| Scopes available | `workers`, `ai-gateway` | Must have correct scope for access | +| Scope | Account-level | Can be reused across multiple Workers | +| Access method | `await env.BINDING.get()` | Async only, throws on error | +| Management | Centralized | Via secrets-store commands | +| Local dev | Separate local secrets | Use without `--remote` flag | +| Regional availability | Global except China Network | Unavailable in China Network | + +See: [configuration.md](./configuration.md), [api.md](./api.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/secrets-store/patterns.md b/plugins/cloudflare/skills/cloudflare/references/secrets-store/patterns.md new file mode 100644 index 0000000..afac998 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/secrets-store/patterns.md @@ -0,0 +1,207 @@ +# Patterns + +## Secret Rotation + +Zero-downtime rotation with versioned naming (`api_key_v1`, `api_key_v2`): + +```typescript +interface Env { + PRIMARY_KEY: { get(): Promise }; + FALLBACK_KEY?: { get(): Promise }; +} + +async function fetchWithAuth(url: string, key: string) { + return fetch(url, { headers: { "Authorization": `Bearer ${key}` } }); +} + +export default { + async fetch(request: Request, env: Env): Promise { + let resp = await fetchWithAuth("https://api.example.com", await env.PRIMARY_KEY.get()); + + // Fallback during rotation + if (!resp.ok && env.FALLBACK_KEY) { + resp = await fetchWithAuth("https://api.example.com", await env.FALLBACK_KEY.get()); + } + + return resp; + } +} +``` + +Workflow: Create `api_key_v2` → add fallback binding → deploy → swap primary → deploy → remove `v1` + +## Encryption with KV + +```typescript +interface Env { + CACHE: KVNamespace; + ENCRYPTION_KEY: { get(): Promise }; +} + +async function encryptValue(value: string, key: string): Promise { + const enc = new TextEncoder(); + const keyMaterial = await crypto.subtle.importKey( + "raw", enc.encode(key), { name: "AES-GCM" }, false, ["encrypt"] + ); + const iv = crypto.getRandomValues(new Uint8Array(12)); + const encrypted = await crypto.subtle.encrypt( + { name: "AES-GCM", iv }, keyMaterial, enc.encode(value) + ); + + const combined = new Uint8Array(iv.length + encrypted.byteLength); + combined.set(iv); + combined.set(new Uint8Array(encrypted), iv.length); + return btoa(String.fromCharCode(...combined)); +} + +export default { + async fetch(request: Request, env: Env): Promise { + const key = await env.ENCRYPTION_KEY.get(); + const encrypted = await encryptValue("sensitive-data", key); + await env.CACHE.put("user:123:data", encrypted); + return Response.json({ ok: true }); + } +} +``` + +## HMAC Signing + +```typescript +interface Env { + HMAC_SECRET: { get(): Promise }; +} + +async function signRequest(data: string, secret: string): Promise { + const enc = new TextEncoder(); + const key = await crypto.subtle.importKey( + "raw", enc.encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"] + ); + const sig = await crypto.subtle.sign("HMAC", key, enc.encode(data)); + return btoa(String.fromCharCode(...new Uint8Array(sig))); +} + +export default { + async fetch(request: Request, env: Env): Promise { + const secret = await env.HMAC_SECRET.get(); + const payload = await request.text(); + const signature = await signRequest(payload, secret); + return Response.json({ signature }); + } +} +``` + +## Audit & Monitoring + +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext) { + const startTime = Date.now(); + try { + const apiKey = await env.API_KEY.get(); + const resp = await fetch("https://api.example.com", { + headers: { "Authorization": `Bearer ${apiKey}` } + }); + + ctx.waitUntil( + fetch("https://log.example.com/log", { + method: "POST", + body: JSON.stringify({ + event: "secret_used", + secret_name: "API_KEY", + timestamp: new Date().toISOString(), + duration_ms: Date.now() - startTime, + success: resp.ok + }) + }) + ); + return resp; + } catch (error) { + ctx.waitUntil( + fetch("https://log.example.com/log", { + method: "POST", + body: JSON.stringify({ + event: "secret_access_failed", + secret_name: "API_KEY", + error: error instanceof Error ? error.message : "Unknown" + }) + }) + ); + return new Response("Error", { status: 500 }); + } + } +} +``` + +## Migration from Worker Secrets + +Change `env.SECRET` (direct) to `await env.SECRET.get()` (async). + +Steps: +1. Create in Secrets Store: `wrangler secrets-store secret create --name API_KEY --scopes workers --remote` +2. Add binding to `wrangler.jsonc`: `{"binding": "API_KEY", "store_id": "abc123", "secret_name": "api_key"}` +3. Update code: `const key = await env.API_KEY.get();` +4. Test staging, deploy +5. Remove old: `wrangler secret delete API_KEY` + +## Sharing Across Workers + +Same secret, different binding names: + +```jsonc +// worker-1: binding="SHARED_DB", secret_name="postgres_url" +// worker-2: binding="DB_CONN", secret_name="postgres_url" +``` + +## JSON Secret Parsing + +Store structured config as JSON secrets: + +```typescript +interface Env { + DB_CONFIG: { get(): Promise }; +} + +interface DbConfig { + host: string; + port: number; + username: string; + password: string; +} + +export default { + async fetch(request: Request, env: Env): Promise { + try { + const configStr = await env.DB_CONFIG.get(); + const config: DbConfig = JSON.parse(configStr); + + // Use parsed config + const dbUrl = `postgres://${config.username}:${config.password}@${config.host}:${config.port}`; + + return Response.json({ connected: true }); + } catch (error) { + if (error instanceof SyntaxError) { + return new Response("Invalid config JSON", { status: 500 }); + } + throw error; + } + } +} +``` + +Store JSON secret: + +```bash +echo '{"host":"db.example.com","port":5432,"username":"app","password":"secret"}' | \ + wrangler secrets-store secret create \ + --name DB_CONFIG --scopes workers --remote +``` + +## Integration + +### Service Bindings + +Auth Worker signs JWT with Secrets Store; API Worker verifies via service binding. + +See: [workers](../workers/) for service binding patterns. + +See: [api.md](./api.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/smart-placement/README.md b/plugins/cloudflare/skills/cloudflare/references/smart-placement/README.md new file mode 100644 index 0000000..e8b1041 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/smart-placement/README.md @@ -0,0 +1,138 @@ +# Cloudflare Workers Smart Placement + +Automatic workload placement optimization to minimize latency by running Workers closer to backend infrastructure rather than end users. + +## Core Concept + +Smart Placement automatically analyzes Worker request duration across Cloudflare's global network and intelligently routes requests to optimal data center locations. Instead of defaulting to the location closest to the end user, Smart Placement can forward requests to locations closer to backend infrastructure when this reduces overall request duration. + +### When to Use + +**Enable Smart Placement when:** +- Worker makes multiple round trips to backend services/databases +- Backend infrastructure is geographically concentrated +- Request duration dominated by backend latency rather than network latency from user +- Running backend logic in Workers (APIs, data aggregation, SSR with DB calls) +- Worker uses `fetch` handler (not RPC methods) + +**Do NOT enable for:** +- Workers serving only static content or cached responses +- Workers without significant backend communication +- Pure edge logic (auth checks, redirects, simple transformations) +- Workers without fetch event handlers +- Workers with RPC methods or named entrypoints (only `fetch` handlers are affected) +- Pages/Assets Workers with `run_worker_first = true` (degrades asset serving) + +### Decision Tree + +``` +Does your Worker have a fetch handler? +├─ No → Smart Placement won't work (skip) +└─ Yes + │ + Does it make multiple backend calls (DB/API)? + ├─ No → Don't enable (won't help) + └─ Yes + │ + Is backend geographically concentrated? + ├─ No (globally distributed) → Probably won't help + └─ Yes or uncertain + │ + Does it serve static assets with run_worker_first=true? + ├─ Yes → Don't enable (will hurt performance) + └─ No → Enable Smart Placement + │ + After 15min, check placement_status + ├─ SUCCESS → Monitor metrics + ├─ INSUFFICIENT_INVOCATIONS → Need more traffic + └─ UNSUPPORTED_APPLICATION → Disable (hurting performance) +``` + +### Key Architecture Pattern + +**Recommended:** Split full-stack applications into separate Workers: +``` +User → Frontend Worker (at edge, close to user) + ↓ Service Binding + Backend Worker (Smart Placement enabled, close to DB/API) + ↓ + Database/Backend Service +``` + +This maintains fast, reactive frontends while optimizing backend latency. + +## Quick Start + +```jsonc +// wrangler.jsonc +{ + "placement": { + "mode": "smart" // or "off" to explicitly disable + } +} +``` + +Deploy and wait 15 minutes for analysis. Check status via API or dashboard metrics. + +**To disable:** Set `"mode": "off"` or remove `placement` field entirely (both equivalent). + +## Requirements + +- Wrangler 2.20.0+ +- Analysis time: Up to 15 minutes after enabling +- Traffic requirements: Consistent traffic from multiple global locations +- Available on all Workers plans (Free, Paid, Enterprise) + +## Placement Status Values + +```typescript +type PlacementStatus = + | undefined // Not yet analyzed + | 'SUCCESS' // Successfully optimized + | 'INSUFFICIENT_INVOCATIONS' // Not enough traffic + | 'UNSUPPORTED_APPLICATION'; // Made Worker slower (reverted) +``` + +## CLI Commands + +```bash +# Deploy with Smart Placement +wrangler deploy + +# Check placement status +curl -H "Authorization: Bearer $TOKEN" \ + https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/services/$WORKER_NAME \ + | jq .result.placement_status + +# Monitor +wrangler tail your-worker-name --header cf-placement +``` + +## Reading Order + +**First time?** Start here: +1. This README - understand core concepts and when to use Smart Placement +2. [configuration.md](./configuration.md) - set up wrangler.jsonc and understand limitations +3. [patterns.md](./patterns.md) - see practical examples for your use case +4. [api.md](./api.md) - monitor and verify Smart Placement is working +5. [gotchas.md](./gotchas.md) - troubleshoot common issues + +**Quick lookup:** +- "Should I enable Smart Placement?" → See "When to Use" above +- "How do I configure it?" → [configuration.md](./configuration.md) +- "How do I split frontend/backend?" → [patterns.md](./patterns.md) +- "Why isn't it working?" → [gotchas.md](./gotchas.md) + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc setup, mode values, validation rules +- [api.md](./api.md) - Placement Status API, cf-placement header, monitoring +- [patterns.md](./patterns.md) - Frontend/backend split, database workers, SSR patterns +- [gotchas.md](./gotchas.md) - Troubleshooting INSUFFICIENT_INVOCATIONS, performance issues + +## See Also + +- [workers](../workers/) - Worker runtime and fetch handlers +- [d1](../d1/) - D1 database that benefits from Smart Placement +- [durable-objects](../durable-objects/) - Durable Objects with backend logic +- [bindings](../bindings/) - Service bindings for frontend/backend split diff --git a/plugins/cloudflare/skills/cloudflare/references/smart-placement/api.md b/plugins/cloudflare/skills/cloudflare/references/smart-placement/api.md new file mode 100644 index 0000000..6608985 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/smart-placement/api.md @@ -0,0 +1,183 @@ +# Smart Placement API + +## Placement Status API + +Query Worker placement status via Cloudflare API: + +```bash +curl -X GET "https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/workers/services/{WORKER_NAME}" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" +``` + +Response includes `placement_status` field: + +```typescript +type PlacementStatus = + | undefined // Not yet analyzed + | 'SUCCESS' // Successfully optimized + | 'INSUFFICIENT_INVOCATIONS' // Not enough traffic + | 'UNSUPPORTED_APPLICATION'; // Made Worker slower (reverted) +``` + +## Status Meanings + +**`undefined` (not present)** +- Worker not yet analyzed +- Always runs at default edge location closest to user + +**`SUCCESS`** +- Analysis complete, Smart Placement active +- Worker runs in optimal location (may be edge or remote) + +**`INSUFFICIENT_INVOCATIONS`** +- Not enough requests to make placement decision +- Requires consistent multi-region traffic +- Always runs at default edge location + +**`UNSUPPORTED_APPLICATION`** (rare, <1% of Workers) +- Smart Placement made Worker slower +- Placement decision reverted +- Always runs at edge location +- Won't be re-analyzed until redeployed + +## cf-placement Header (Beta) + +Smart Placement adds response header indicating routing decision: + +```typescript +// Remote placement (Smart Placement routed request) +"cf-placement: remote-LHR" // Routed to London + +// Local placement (default edge routing) +"cf-placement: local-EWR" // Stayed at Newark edge +``` + +Format: `{placement-type}-{IATA-code}` +- `remote-*` = Smart Placement routed to remote location +- `local-*` = Stayed at default edge location +- IATA code = nearest airport to data center + +**Warning:** Beta feature, may be removed before GA. + +## Detecting Smart Placement in Code + +**Note:** `cf-placement` header is a beta feature and may change or be removed. + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const placementHeader = request.headers.get('cf-placement'); + + if (placementHeader?.startsWith('remote-')) { + const location = placementHeader.split('-')[1]; + console.log(`Smart Placement routed to ${location}`); + } else if (placementHeader?.startsWith('local-')) { + const location = placementHeader.split('-')[1]; + console.log(`Running at edge location ${location}`); + } + + return new Response('OK'); + } +} satisfies ExportedHandler; +``` + +## Request Duration Metrics + +Available in Cloudflare dashboard when Smart Placement enabled: + +**Workers & Pages → [Your Worker] → Metrics → Request Duration** + +Shows histogram comparing: +- Request duration WITH Smart Placement (99% of traffic) +- Request duration WITHOUT Smart Placement (1% baseline) + +**Request Duration vs Execution Duration:** +- **Request duration:** Total time from request arrival to response delivery (includes network latency) +- **Execution duration:** Time Worker code actively executing (excludes network waits) + +Use request duration to measure Smart Placement impact. + +### Interpreting Metrics + +| Metric Comparison | Interpretation | Action | +|-------------------|----------------|--------| +| WITH < WITHOUT | Smart Placement helping | Keep enabled | +| WITH ≈ WITHOUT | Neutral impact | Consider disabling to free resources | +| WITH > WITHOUT | Smart Placement hurting | Disable with `mode: "off"` | + +**Why Smart Placement might hurt performance:** +- Worker primarily serves static assets or cached content +- Backend services are globally distributed (no single optimal location) +- Worker has minimal backend communication +- Using Pages with `assets.run_worker_first = true` + +**Typical improvements when Smart Placement helps:** +- 20-50% reduction in request duration for database-heavy Workers +- 30-60% reduction for Workers making multiple backend API calls +- Larger improvements when backend is geographically concentrated + +## Monitoring Commands + +```bash +# Tail Worker logs +wrangler tail your-worker-name + +# Tail with filters +wrangler tail your-worker-name --status error +wrangler tail your-worker-name --header cf-placement + +# Check placement status via API +curl -H "Authorization: Bearer $TOKEN" \ + https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/services/$WORKER_NAME \ + | jq .result.placement_status +``` + +## TypeScript Types + +```typescript +// Placement status returned by API (field may be absent) +type PlacementStatus = + | 'SUCCESS' + | 'INSUFFICIENT_INVOCATIONS' + | 'UNSUPPORTED_APPLICATION' + | undefined; + +// Placement configuration in wrangler.jsonc +type PlacementMode = 'smart' | 'off'; + +interface PlacementConfig { + mode: PlacementMode; + // Legacy fields (deprecated/removed): + // hint?: string; // REMOVED - no longer supported +} + +// Explicit placement (separate feature from Smart Placement) +interface ExplicitPlacementConfig { + region?: string; + host?: string; + hostname?: string; + // Cannot combine with mode field +} + +// Worker metadata from API response +interface WorkerMetadata { + placement?: PlacementConfig | ExplicitPlacementConfig; + placement_status?: PlacementStatus; +} + +// Service Binding for backend Worker +interface Env { + BACKEND_SERVICE: Fetcher; // Service Binding to backend Worker + DATABASE: D1Database; +} + +// Example Worker with Service Binding +export default { + async fetch(request: Request, env: Env): Promise { + // Forward to backend Worker with Smart Placement enabled + const response = await env.BACKEND_SERVICE.fetch(request); + return response; + } +} satisfies ExportedHandler; +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/smart-placement/configuration.md b/plugins/cloudflare/skills/cloudflare/references/smart-placement/configuration.md new file mode 100644 index 0000000..4f506ac --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/smart-placement/configuration.md @@ -0,0 +1,196 @@ +# Smart Placement Configuration + +## wrangler.jsonc Setup + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "placement": { + "mode": "smart" + } +} +``` + +## Placement Mode Values + +| Mode | Behavior | +|------|----------| +| `"smart"` | Enable Smart Placement - automatic optimization based on traffic analysis | +| `"off"` | Explicitly disable Smart Placement - always run at edge closest to user | +| Not specified | Default behavior - run at edge closest to user (same as `"off"`) | + +**Note:** Smart Placement vs Explicit Placement are separate features. Smart Placement (`mode: "smart"`) uses automatic analysis. For manual placement control, see explicit placement options (`region`, `host`, `hostname` fields - not covered in this reference). + +## Frontend + Backend Split Configuration + +### Frontend Worker (No Smart Placement) + +```jsonc +// frontend-worker/wrangler.jsonc +{ + "name": "frontend", + "main": "frontend-worker.ts", + // No "placement" - runs at edge + "services": [ + { + "binding": "BACKEND", + "service": "backend-api" + } + ] +} +``` + +### Backend Worker (Smart Placement Enabled) + +```jsonc +// backend-api/wrangler.jsonc +{ + "name": "backend-api", + "main": "backend-worker.ts", + "placement": { + "mode": "smart" + }, + "d1_databases": [ + { + "binding": "DATABASE", + "database_id": "xxx" + } + ] +} +``` + +## Requirements & Limitations + +### Requirements +- **Wrangler version:** 2.20.0+ +- **Analysis time:** Up to 15 minutes +- **Traffic requirements:** Consistent multi-location traffic +- **Workers plan:** All plans (Free, Paid, Enterprise) + +### What Smart Placement Affects + +**CRITICAL LIMITATION - Smart Placement ONLY Affects `fetch` Handlers:** + +Smart Placement is fundamentally limited to Workers with default `fetch` handlers. This is a key architectural constraint. + +- ✅ **Affects:** `fetch` event handlers ONLY (the default export's fetch method) +- ❌ **Does NOT affect:** + - RPC methods (Service Bindings with `WorkerEntrypoint` - see example below) + - Named entrypoints (exports other than `default`) + - Workers without `fetch` handlers + - Queue consumers, scheduled handlers, or other event types + +**Example - Smart Placement ONLY affects `fetch`:** +```typescript +// ✅ Smart Placement affects this: +export default { + async fetch(request: Request, env: Env): Promise { + // This runs close to backend when Smart Placement enabled + const data = await env.DATABASE.prepare('SELECT * FROM users').all(); + return Response.json(data); + } +} + +// ❌ Smart Placement DOES NOT affect these: +export class MyRPC extends WorkerEntrypoint { + async myMethod() { + // This ALWAYS runs at edge, Smart Placement has NO EFFECT + const data = await this.env.DATABASE.prepare('SELECT * FROM users').all(); + return data; + } +} + +export async function scheduled(event: ScheduledEvent, env: Env) { + // NOT affected by Smart Placement +} +``` + +**Consequence:** If your backend logic uses RPC methods (`WorkerEntrypoint`), Smart Placement cannot optimize those calls. You must use fetch-based patterns for Smart Placement to work. + +**Solution:** Convert RPC methods to fetch endpoints, or use a wrapper Worker with `fetch` handler that calls your backend RPC (though this adds latency). + +### Baseline Traffic +Smart Placement automatically routes 1% of requests WITHOUT optimization as baseline for performance comparison. + +### Validation Rules + +**Mutually exclusive fields:** +- `mode` cannot be used with explicit placement fields (`region`, `host`, `hostname`) +- Choose either Smart Placement OR explicit placement, not both + +```jsonc +// ✅ Valid - Smart Placement +{ "placement": { "mode": "smart" } } + +// ✅ Valid - Explicit Placement (different feature) +{ "placement": { "region": "us-east1" } } + +// ❌ Invalid - Cannot combine +{ "placement": { "mode": "smart", "region": "us-east1" } } +``` + +## Dashboard Configuration + +**Workers & Pages** → Select Worker → **Settings** → **General** → **Placement: Smart** → Wait 15min → Check **Metrics** + +## TypeScript Types + +```typescript +interface Env { + BACKEND: Fetcher; + DATABASE: D1Database; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const data = await env.DATABASE.prepare('SELECT * FROM table').all(); + return Response.json(data); + } +} satisfies ExportedHandler; +``` + +## Cloudflare Pages/Assets Warning + +**CRITICAL PERFORMANCE ISSUE:** Enabling Smart Placement with `assets.run_worker_first = true` in Pages projects **severely degrades asset serving performance**. This is one of the most common misconfigurations. + +**Why this is bad:** +- Smart Placement routes ALL requests (including static assets) away from edge to remote locations +- Static assets (HTML, CSS, JS, images) should ALWAYS be served from edge closest to user +- Result: 2-5x slower asset loading times, poor user experience + +**Problem:** Smart Placement routes asset requests away from edge, but static assets should always be served from edge closest to user. + +**Solutions (in order of preference):** +1. **Recommended:** Split into separate Workers (frontend at edge + backend with Smart Placement) +2. Set `"mode": "off"` to explicitly disable Smart Placement for Pages/Assets Workers +3. Use `assets.run_worker_first = false` (serves assets first, bypasses Worker for static content) + +```jsonc +// ❌ BAD - Degrades asset performance by 2-5x +{ + "name": "pages-app", + "placement": { "mode": "smart" }, + "assets": { "run_worker_first": true } +} + +// ✅ GOOD - Frontend at edge, backend optimized +// frontend-worker/wrangler.jsonc +{ + "name": "frontend", + "assets": { "run_worker_first": true } + // No placement - runs at edge +} + +// backend-worker/wrangler.jsonc +{ + "name": "backend-api", + "placement": { "mode": "smart" }, + "d1_databases": [{ "binding": "DB", "database_id": "xxx" }] +} +``` + +**Key takeaway:** Never enable Smart Placement on Workers that serve static assets with `run_worker_first = true`. + +## Local Development + +Smart Placement does NOT work in `wrangler dev` (local only). Test by deploying: `wrangler deploy --env staging` diff --git a/plugins/cloudflare/skills/cloudflare/references/smart-placement/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/smart-placement/gotchas.md new file mode 100644 index 0000000..dc94e9b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/smart-placement/gotchas.md @@ -0,0 +1,174 @@ +# Smart Placement Gotchas + +## Common Errors + +### "INSUFFICIENT_INVOCATIONS" + +**Cause:** Not enough traffic for Smart Placement to analyze +**Solution:** +- Ensure Worker receives consistent global traffic +- Wait longer (analysis takes up to 15 minutes) +- Send test traffic from multiple global locations +- Check Worker has fetch event handler + +### "UNSUPPORTED_APPLICATION" + +**Cause:** Smart Placement made Worker slower rather than faster +**Reasons:** +- Worker doesn't make backend calls (runs faster at edge) +- Backend calls are cached (network latency to user more important) +- Backend service has good global distribution +- Worker serves static assets or Pages content + +**Solutions:** +- Disable Smart Placement: `{ "placement": { "mode": "off" } }` +- Review whether Worker actually benefits from Smart Placement +- Consider caching strategy to reduce backend calls +- For Pages/Assets Workers, use separate backend Worker with Smart Placement + +### "No request duration metrics" + +**Cause:** Smart Placement not enabled, insufficient time passed, insufficient traffic, or analysis incomplete +**Solution:** +- Ensure Smart Placement enabled in config +- Wait 15+ minutes after deployment +- Verify Worker has sufficient traffic +- Check `placement_status` is `SUCCESS` + +### "cf-placement header missing" + +**Cause:** Smart Placement not enabled, beta feature removed, or Worker not analyzed yet +**Solution:** Verify Smart Placement enabled, wait for analysis (15min), check if beta feature still available + +## Pages/Assets + Smart Placement Performance Degradation + +**Problem:** Static assets load 2-5x slower when Smart Placement enabled with `run_worker_first = true`. + +**Cause:** Smart Placement routes ALL requests (including static assets like HTML, CSS, JS, images) to remote locations. Static content should ALWAYS be served from edge closest to user. + +**Solution:** Split into separate Workers OR disable Smart Placement: +```jsonc +// ❌ BAD - Assets routed away from user +{ + "name": "pages-app", + "placement": { "mode": "smart" }, + "assets": { "run_worker_first": true } +} + +// ✅ GOOD - Assets at edge, API optimized +// frontend/wrangler.jsonc +{ + "name": "frontend", + "assets": { "run_worker_first": true } + // No placement field - stays at edge +} + +// backend/wrangler.jsonc +{ + "name": "backend-api", + "placement": { "mode": "smart" } +} +``` + +This is one of the most common and impactful Smart Placement misconfigurations. + +## Monolithic Full-Stack Worker + +**Problem:** Frontend and backend logic in single Worker with Smart Placement enabled. + +**Cause:** Smart Placement optimizes for backend latency but increases user-facing response time. + +**Solution:** Split into two Workers: +```jsonc +// frontend/wrangler.jsonc +{ + "name": "frontend", + "placement": { "mode": "off" }, // Explicit: stay at edge + "services": [{ "binding": "BACKEND", "service": "backend-api" }] +} + +// backend/wrangler.jsonc +{ + "name": "backend-api", + "placement": { "mode": "smart" }, + "d1_databases": [{ "binding": "DB", "database_id": "xxx" }] +} +``` + +## Local Development Confusion + +**Issue:** Smart Placement doesn't work in `wrangler dev`. + +**Explanation:** Smart Placement only activates in production deployments, not local development. + +**Solution:** Test Smart Placement in staging environment: `wrangler deploy --env staging` + +## Baseline Traffic & Analysis Time + +**Note:** Smart Placement routes 1% of requests WITHOUT optimization for comparison (expected). + +**Analysis time:** Up to 15 minutes. During analysis, Worker runs at edge. Monitor `placement_status`. + +## RPC Methods Not Affected (Critical Limitation) + +**Problem:** Enabled Smart Placement on backend but RPC calls still slow. + +**Cause:** Smart Placement ONLY affects `fetch` handlers. RPC methods (Service Bindings with `WorkerEntrypoint`) are NEVER affected. + +**Why:** RPC bypasses `fetch` handler - Smart Placement can only route `fetch` requests. + +**Solution:** Convert to fetch-based Service Bindings: + +```typescript +// ❌ RPC - Smart Placement has NO EFFECT +export class BackendRPC extends WorkerEntrypoint { + async getData() { + // ALWAYS runs at edge + return await this.env.DATABASE.prepare('SELECT * FROM table').all(); + } +} + +// ✅ Fetch - Smart Placement WORKS +export default { + async fetch(request: Request, env: Env): Promise { + // Runs close to DATABASE when Smart Placement enabled + const data = await env.DATABASE.prepare('SELECT * FROM table').all(); + return Response.json(data); + } +} +``` + +## Requirements + +- **Wrangler 2.20.0+** required +- **Consistent multi-region traffic** needed for analysis +- **Only affects fetch handlers** - RPC methods and named entrypoints not affected + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Analysis time | Up to 15 minutes | After enabling | +| Baseline traffic | 1% | Routed without optimization | +| Min Wrangler version | 2.20.0+ | Required | +| Traffic requirement | Multi-region | Consistent needed | + +## Disabling Smart Placement + +```jsonc +{ "placement": { "mode": "off" } } // Explicit disable +// OR remove "placement" field entirely (same effect) +``` + +Both behaviors identical - Worker runs at edge closest to user. + +## When NOT to Use Smart Placement + +- Workers serving only static content or cached responses +- Workers without significant backend communication +- Pure edge logic (auth checks, redirects, simple transformations) +- Workers without fetch event handlers +- Pages/Assets Workers with `run_worker_first = true` +- Workers using RPC methods instead of fetch handlers + +These scenarios won't benefit and may perform worse with Smart Placement. diff --git a/plugins/cloudflare/skills/cloudflare/references/smart-placement/patterns.md b/plugins/cloudflare/skills/cloudflare/references/smart-placement/patterns.md new file mode 100644 index 0000000..40dc4dd --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/smart-placement/patterns.md @@ -0,0 +1,183 @@ +# Smart Placement Patterns + +## Backend Worker with Database Access + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const user = await env.DATABASE.prepare('SELECT * FROM users WHERE id = ?').bind(userId).first(); + const orders = await env.DATABASE.prepare('SELECT * FROM orders WHERE user_id = ?').bind(userId).all(); + return Response.json({ user, orders }); + } +}; +``` + +```jsonc +{ "placement": { "mode": "smart" }, "d1_databases": [{ "binding": "DATABASE", "database_id": "xxx" }] } +``` + +## Frontend + Backend Split (Service Bindings) + +**Frontend:** Runs at edge for fast user response +**Backend:** Smart Placement runs close to database + +```typescript +// Frontend Worker - routes requests to backend +interface Env { + BACKEND: Fetcher; // Service Binding to backend Worker +} + +export default { + async fetch(request: Request, env: Env): Promise { + if (new URL(request.url).pathname.startsWith('/api/')) { + return env.BACKEND.fetch(request); // Forward to backend + } + return new Response('Frontend content'); + } +}; + +// Backend Worker - database operations +interface BackendEnv { + DATABASE: D1Database; +} + +export default { + async fetch(request: Request, env: BackendEnv): Promise { + const data = await env.DATABASE.prepare('SELECT * FROM table').all(); + return Response.json(data); + } +}; +``` + +**CRITICAL:** Use fetch-based Service Bindings (shown above). If using RPC with `WorkerEntrypoint`, Smart Placement will NOT optimize those method calls - only `fetch` handlers are affected. + +**RPC vs Fetch - CRITICAL:** Smart Placement ONLY works with fetch-based bindings, NOT RPC. + +```typescript +// ❌ RPC - Smart Placement has NO EFFECT on backend RPC methods +export class BackendRPC extends WorkerEntrypoint { + async getData() { + // ALWAYS runs at edge, Smart Placement ignored + return await this.env.DATABASE.prepare('SELECT * FROM table').all(); + } +} + +// ✅ Fetch - Smart Placement WORKS +export default { + async fetch(request: Request, env: Env): Promise { + // Runs close to DATABASE when Smart Placement enabled + const data = await env.DATABASE.prepare('SELECT * FROM table').all(); + return Response.json(data); + } +}; +``` + +## External API Integration + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const apiUrl = 'https://api.partner.com'; + const headers = { 'Authorization': `Bearer ${env.API_KEY}` }; + + const [profile, transactions] = await Promise.all([ + fetch(`${apiUrl}/profile`, { headers }), + fetch(`${apiUrl}/transactions`, { headers }) + ]); + + return Response.json({ + profile: await profile.json(), + transactions: await transactions.json() + }); + } +}; +``` + +## SSR / API Gateway Pattern + +```typescript +// Frontend (edge) - auth/routing close to user +export default { + async fetch(request: Request, env: Env) { + if (!request.headers.get('Authorization')) { + return new Response('Unauthorized', { status: 401 }); + } + const data = await env.BACKEND.fetch(request); + return new Response(renderPage(await data.json()), { + headers: { 'Content-Type': 'text/html' } + }); + } +}; + +// Backend (Smart Placement) - DB operations close to data +export default { + async fetch(request: Request, env: Env) { + const data = await env.DATABASE.prepare('SELECT * FROM pages WHERE id = ?').bind(pageId).first(); + return Response.json(data); + } +}; +``` + +## Durable Objects with Smart Placement + +**Key principle:** Smart Placement does NOT control WHERE Durable Objects run. DOs always run in their designated region (based on jurisdiction or smart location hints). + +**What Smart Placement DOES affect:** The location of the coordinator Worker's `fetch` handler that makes calls to multiple DOs. + +**Pattern:** Enable Smart Placement on coordinator Worker that aggregates data from multiple DOs: + +```typescript +// Worker with Smart Placement - aggregates data from multiple DOs +export default { + async fetch(request: Request, env: Env): Promise { + const userId = new URL(request.url).searchParams.get('user'); + + // Get DO stubs + const userDO = env.USER_DO.get(env.USER_DO.idFromName(userId)); + const analyticsID = env.ANALYTICS_DO.idFromName(`analytics-${userId}`); + const analyticsDO = env.ANALYTICS_DO.get(analyticsID); + + // Fetch from multiple DOs + const [userData, analyticsData] = await Promise.all([ + userDO.fetch(new Request('https://do/profile')), + analyticsDO.fetch(new Request('https://do/stats')) + ]); + + return Response.json({ + user: await userData.json(), + analytics: await analyticsData.json() + }); + } +}; +``` + +```jsonc +// wrangler.jsonc +{ + "placement": { "mode": "smart" }, + "durable_objects": { + "bindings": [ + { "name": "USER_DO", "class_name": "UserDO" }, + { "name": "ANALYTICS_DO", "class_name": "AnalyticsDO" } + ] + } +} +``` + +**When this helps:** +- Worker's `fetch` handler runs closer to DO regions, reducing network latency for multiple DO calls +- Most beneficial when DOs are geographically concentrated or in specific jurisdictions +- Helps when coordinator makes many sequential or parallel DO calls + +**When this DOESN'T help:** +- DOs are globally distributed (no single optimal Worker location) +- Worker only calls a single DO +- DO calls are infrequent or cached + +## Best Practices + +- Split full-stack apps: frontend at edge, backend with Smart Placement +- Use fetch-based Service Bindings (not RPC) +- Enable for backend logic: APIs, data aggregation, DB operations +- Don't enable for: static content, edge logic, RPC methods, Pages with `run_worker_first` +- Wait 15+ min for analysis, verify `placement_status = SUCCESS` diff --git a/plugins/cloudflare/skills/cloudflare/references/snippets/README.md b/plugins/cloudflare/skills/cloudflare/references/snippets/README.md new file mode 100644 index 0000000..a09a1c4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/snippets/README.md @@ -0,0 +1,68 @@ +# Cloudflare Snippets Skill Reference + +## Description +Expert guidance for **Cloudflare Snippets ONLY** - a lightweight JavaScript-based edge logic platform for modifying HTTP requests and responses. Snippets run as part of the Ruleset Engine and are included at no additional cost on paid plans (Pro, Business, Enterprise). + +## What Are Snippets? +Snippets are JavaScript functions executed at the edge as part of Cloudflare's Ruleset Engine. Key characteristics: +- **Execution time**: 5ms CPU limit per request +- **Size limit**: 32KB per snippet +- **Runtime**: V8 isolate (subset of Workers APIs) +- **Subrequests**: 2-5 fetch calls depending on plan +- **Cost**: Included with Pro/Business/Enterprise plans + +## Snippets vs Workers Decision Matrix + +| Factor | Choose Snippets If... | Choose Workers If... | +|--------|----------------------|---------------------| +| **Complexity** | Simple request/response modifications | Complex business logic, routing, middleware | +| **Execution time** | <5ms sufficient | Need >5ms or variable time | +| **Subrequests** | 2-5 fetch calls sufficient | Need >5 subrequests or complex orchestration | +| **Code size** | <32KB sufficient | Need >32KB or npm dependencies | +| **Cost** | Want zero additional cost | Can afford $5/mo + usage | +| **APIs** | Need basic fetch, headers, URL | Need KV, D1, R2, Durable Objects, cron triggers | +| **Deployment** | Need rule-based triggers | Want custom routing logic | + +**Rule of thumb**: Use Snippets for modifications, Workers for applications. + +## Execution Model +1. Request arrives at Cloudflare edge +2. Ruleset Engine evaluates snippet rules (filter expressions) +3. If rule matches, snippet executes within 5ms limit +4. Modified request/response continues through pipeline +5. Response returned to client + +Snippets execute synchronously in the request path - performance is critical. + +## Reading Order +1. **[configuration.md](configuration.md)** - Start here: setup, deployment methods (Dashboard/API/Terraform) +2. **[api.md](api.md)** - Core APIs: Request, Response, headers, `request.cf` properties +3. **[patterns.md](patterns.md)** - Real-world examples: geo-routing, A/B tests, security headers +4. **[gotchas.md](gotchas.md)** - Troubleshooting: common errors, performance tips, API limitations + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, configuration +- **[api.md](api.md)** - API endpoints, methods, interfaces +- **[patterns.md](patterns.md)** - Common patterns, use cases, examples +- **[gotchas.md](gotchas.md)** - Troubleshooting, best practices, limitations + +## Quick Start +```javascript +// Snippet: Add security headers +export default { + async fetch(request) { + const response = await fetch(request); + const newResponse = new Response(response.body, response); + newResponse.headers.set("X-Frame-Options", "DENY"); + newResponse.headers.set("X-Content-Type-Options", "nosniff"); + return newResponse; + } +} +``` + +Deploy via Dashboard (Rules → Snippets) or API/Terraform. See configuration.md for details. + +## See Also + +- [Cloudflare Docs](https://developers.cloudflare.com/rules/snippets/) diff --git a/plugins/cloudflare/skills/cloudflare/references/snippets/api.md b/plugins/cloudflare/skills/cloudflare/references/snippets/api.md new file mode 100644 index 0000000..76a5a4b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/snippets/api.md @@ -0,0 +1,198 @@ +# Snippets API Reference + +## Request Object + +### HTTP Properties +```javascript +request.method // GET, POST, PUT, DELETE, etc. +request.url // Full URL string +request.headers // Headers object +request.body // ReadableStream (for POST/PUT) +request.cf // Cloudflare properties (see below) +``` + +### URL Operations +```javascript +const url = new URL(request.url); +url.hostname // "example.com" +url.pathname // "/path/to/page" +url.search // "?query=value" +url.searchParams.get("q") // "value" +url.searchParams.set("q", "new") +url.searchParams.delete("q") +``` + +### Header Operations +```javascript +// Read headers +request.headers.get("User-Agent") +request.headers.has("Authorization") +request.headers.getSetCookie() // Get all Set-Cookie headers + +// Modify headers (create new request) +const modifiedRequest = new Request(request); +modifiedRequest.headers.set("X-Custom", "value") +modifiedRequest.headers.delete("X-Remove") +``` + +### Cloudflare Properties (`request.cf`) +Access Cloudflare-specific metadata about the request: + +```javascript +// Geolocation +request.cf.city // "San Francisco" +request.cf.continent // "NA" +request.cf.country // "US" +request.cf.region // "California" or "CA" +request.cf.regionCode // "CA" +request.cf.postalCode // "94102" +request.cf.latitude // "37.7749" +request.cf.longitude // "-122.4194" +request.cf.timezone // "America/Los_Angeles" +request.cf.metroCode // "807" (DMA code) + +// Network +request.cf.colo // "SFO" (airport code of datacenter) +request.cf.asn // 13335 (ASN number) +request.cf.asOrganization // "Cloudflare, Inc." + +// Bot Management (if enabled) +request.cf.botManagement.score // 1-99 (1=bot, 99=human) +request.cf.botManagement.verified_bot // true/false +request.cf.botManagement.static_resource // true/false + +// TLS/HTTP version +request.cf.tlsVersion // "TLSv1.3" +request.cf.tlsCipher // "AEAD-AES128-GCM-SHA256" +request.cf.httpProtocol // "HTTP/2" + +// Request metadata +request.cf.requestPriority // "weight=192;exclusive=0" +``` + +**Use cases**: Geo-routing, bot detection, security decisions, analytics. + +## Response Object + +### Response Constructors +```javascript +// Plain text +new Response("Hello", { status: 200 }) + +// JSON +Response.json({ key: "value" }, { status: 200 }) + +// HTML +new Response("

Hi

", { + status: 200, + headers: { "Content-Type": "text/html" } +}) + +// Redirect +Response.redirect("https://example.com", 301) // or 302 + +// Stream (pass through) +new Response(response.body, response) +``` + +### Response Headers +```javascript +// Create modified response +const newResponse = new Response(response.body, response); + +// Set/modify headers +newResponse.headers.set("X-Custom", "value") +newResponse.headers.append("Set-Cookie", "session=abc; Path=/") +newResponse.headers.delete("Server") + +// Common headers +newResponse.headers.set("Cache-Control", "public, max-age=3600") +newResponse.headers.set("Content-Type", "application/json") +``` + +### Response Properties +```javascript +response.status // 200, 404, 500, etc. +response.statusText // "OK", "Not Found", etc. +response.headers // Headers object +response.body // ReadableStream +response.ok // true if status 200-299 +response.redirected // true if redirected +``` + +## REST API Operations + +### List Snippets +```bash +GET /zones/{zone_id}/snippets +``` + +### Get Snippet +```bash +GET /zones/{zone_id}/snippets/{snippet_name} +``` + +### Create/Update Snippet +```bash +PUT /zones/{zone_id}/snippets/{snippet_name} +Content-Type: multipart/form-data + +files=@snippet.js +metadata={"main_module":"snippet.js"} +``` + +### Delete Snippet +```bash +DELETE /zones/{zone_id}/snippets/{snippet_name} +``` + +### List Snippet Rules +```bash +GET /zones/{zone_id}/rulesets/phases/http_request_snippets/entrypoint +``` + +### Update Snippet Rules +```bash +PUT /zones/{zone_id}/snippets/snippet_rules +Content-Type: application/json + +{ + "rules": [{ + "description": "Apply snippet", + "enabled": true, + "expression": "http.host eq \"example.com\"", + "snippet_name": "my_snippet" + }] +} +``` + +## Available APIs in Snippets + +### ✅ Supported +- `fetch()` - HTTP requests (2-5 subrequests per plan) +- `Request` / `Response` - Standard Web APIs +- `URL` / `URLSearchParams` - URL manipulation +- `Headers` - Header manipulation +- `TextEncoder` / `TextDecoder` - Text encoding +- `crypto.subtle` - Web Crypto API (hashing, signing) +- `crypto.randomUUID()` - UUID generation + +### ❌ Not Supported in Snippets +- `caches` API - Not available (use Workers) +- `KV`, `D1`, `R2` - Storage APIs (use Workers) +- `Durable Objects` - Stateful objects (use Workers) +- `WebSocket` - WebSocket upgrades (use Workers) +- `HTMLRewriter` - HTML parsing (use Workers) +- `import` statements - No module imports +- `addEventListener` - Use `export default { async fetch() {}` pattern + +## Snippet Structure +```javascript +export default { + async fetch(request) { + // Your logic here + const response = await fetch(request); + return response; // or modified response + } +} +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/snippets/configuration.md b/plugins/cloudflare/skills/cloudflare/references/snippets/configuration.md new file mode 100644 index 0000000..b5bea0f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/snippets/configuration.md @@ -0,0 +1,227 @@ +# Snippets Configuration Guide + +## Configuration Methods + +### 1. Dashboard (GUI) +**Best for**: Quick tests, single snippets, visual rule building + +``` +1. Go to zone → Rules → Snippets +2. Click "Create Snippet" or select template +3. Enter snippet name (a-z, 0-9, _ only, cannot change later) +4. Write JavaScript code (32KB max) +5. Configure snippet rule: + - Expression Builder (visual) or Expression Editor (text) + - Use Ruleset Engine filter expressions +6. Test with Preview/HTTP tabs +7. Deploy or Save as Draft +``` + +### 2. REST API +**Best for**: CI/CD, automation, programmatic management + +```bash +# Create/update snippet +curl "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/snippets/$SNIPPET_NAME" \ + --request PUT \ + --header "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + --form "files=@example.js" \ + --form "metadata={\"main_module\": \"example.js\"}" + +# Create snippet rule +curl "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/snippets/snippet_rules" \ + --request PUT \ + --header "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + --header "Content-Type: application/json" \ + --data '{ + "rules": [ + { + "description": "Trigger snippet on /api paths", + "enabled": true, + "expression": "starts_with(http.request.uri.path, \"/api/\")", + "snippet_name": "api_snippet" + } + ] + }' + +# List snippets +curl "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/snippets" \ + --header "Authorization: Bearer $CLOUDFLARE_API_TOKEN" + +# Delete snippet +curl "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/snippets/$SNIPPET_NAME" \ + --request DELETE \ + --header "Authorization: Bearer $CLOUDFLARE_API_TOKEN" +``` + +### 3. Terraform +**Best for**: Infrastructure-as-code, multi-zone deployments + +```hcl +# Configure Terraform provider +terraform { + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 4.0" + } + } +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +# Create snippet +resource "cloudflare_snippet" "security_headers" { + zone_id = var.zone_id + name = "security_headers" + + main_module = "security_headers.js" + files { + name = "security_headers.js" + content = file("${path.module}/snippets/security_headers.js") + } +} + +# Create snippet rule +resource "cloudflare_snippet_rules" "security_rules" { + zone_id = var.zone_id + + rules { + description = "Apply security headers to all requests" + enabled = true + expression = "true" + snippet_name = cloudflare_snippet.security_headers.name + } +} +``` + +### 4. Pulumi +**Best for**: Multi-cloud IaC, TypeScript/Python/Go workflows + +```typescript +import * as cloudflare from "@pulumi/cloudflare"; +import * as fs from "fs"; + +// Create snippet +const securitySnippet = new cloudflare.Snippet("security-headers", { + zoneId: zoneId, + name: "security_headers", + mainModule: "security_headers.js", + files: [{ + name: "security_headers.js", + content: fs.readFileSync("./snippets/security_headers.js", "utf8"), + }], +}); + +// Create snippet rule +const snippetRule = new cloudflare.SnippetRules("security-rules", { + zoneId: zoneId, + rules: [{ + description: "Apply security headers", + enabled: true, + expression: "true", + snippetName: securitySnippet.name, + }], +}); +``` + +## Filter Expressions + +Snippets use Cloudflare's Ruleset Engine expression language to determine when to execute. + +### Common Expression Patterns + +```javascript +// Host matching +http.host eq "example.com" +http.host in {"example.com" "www.example.com"} +http.host contains "example" + +// Path matching +http.request.uri.path eq "/api/users" +starts_with(http.request.uri.path, "/api/") +ends_with(http.request.uri.path, ".json") +matches(http.request.uri.path, "^/api/v[0-9]+/") + +// Query parameters +http.request.uri.query contains "debug=true" + +// Headers +http.headers["user-agent"] contains "Mobile" +http.headers["accept-language"] eq "en-US" + +// Cookies +http.cookie contains "session=" + +// Geolocation +ip.geoip.country eq "US" +ip.geoip.continent eq "EU" + +// Bot detection (requires Bot Management) +cf.bot_management.score lt 30 + +// Method +http.request.method eq "POST" +http.request.method in {"POST" "PUT" "PATCH"} + +// Combine with logical operators +http.host eq "example.com" and starts_with(http.request.uri.path, "/api/") +ip.geoip.country eq "US" or ip.geoip.country eq "CA" +not http.headers["user-agent"] contains "bot" +``` + +### Expression Functions + +| Function | Example | Description | +|----------|---------|-------------| +| `starts_with()` | `starts_with(http.request.uri.path, "/api/")` | Check prefix | +| `ends_with()` | `ends_with(http.request.uri.path, ".json")` | Check suffix | +| `contains()` | `contains(http.headers["user-agent"], "Mobile")` | Check substring | +| `matches()` | `matches(http.request.uri.path, "^/api/")` | Regex match | +| `lower()` | `lower(http.host) eq "example.com"` | Convert to lowercase | +| `upper()` | `upper(http.headers["x-api-key"])` | Convert to uppercase | +| `len()` | `len(http.request.uri.path) gt 100` | String length | + +## Deployment Workflow + +### Development +1. Write snippet code locally +2. Test syntax with `node snippet.js` or TypeScript compiler +3. Deploy to Dashboard or use API with `Save as Draft` +4. Test with Preview/HTTP tabs in Dashboard +5. Enable rule when ready + +### Production +1. Store snippet code in version control +2. Use Terraform/Pulumi for reproducible deployments +3. Deploy to staging zone first +4. Test with real traffic (use low-traffic subdomain) +5. Apply to production zone +6. Monitor with Analytics/Logpush + +## Limits & Requirements + +| Resource | Limit | Notes | +|----------|-------|-------| +| Snippet size | 32 KB | Per snippet, compressed | +| Snippet name | 64 chars | `a-z`, `0-9`, `_` only, immutable | +| Snippets per zone | 20 | Soft limit, contact support for more | +| Rules per zone | 20 | One rule per snippet typical | +| Expression length | 4096 chars | Per rule expression | + +## Authentication + +### API Token (Recommended) +```bash +# Create token at: https://dash.cloudflare.com/profile/api-tokens +# Required permissions: Zone.Snippets:Edit, Zone.Rules:Edit +export CLOUDFLARE_API_TOKEN="your_token_here" +``` + +### API Key (Legacy) +```bash +export CLOUDFLARE_EMAIL="your@email.com" +export CLOUDFLARE_API_KEY="your_global_api_key" +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/snippets/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/snippets/gotchas.md new file mode 100644 index 0000000..832077e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/snippets/gotchas.md @@ -0,0 +1,86 @@ +# Gotchas & Best Practices + +## Common Errors + +### 1000: "Snippet execution failed" +Runtime error or syntax error. Wrap code in try/catch: +```javascript +try { return await fetch(request); } +catch (error) { return new Response(`Error: ${error.message}`, { status: 500 }); } +``` + +### 1100: "Exceeded execution limit" +Code takes >5ms CPU. Simplify logic or move to Workers. + +### 1201: "Multiple origin fetches" +Call `fetch(request)` exactly once: +```javascript +// ❌ Multiple origin fetches +const r1 = await fetch(request); const r2 = await fetch(request); +// ✅ Single fetch, reuse response +const response = await fetch(request); +``` + +### 1202: "Subrequest limit exceeded" +Pro: 2 subrequests, Business/Enterprise: 5. Reduce fetch calls. + +### "Cannot set property on immutable object" +Clone before modifying: +```javascript +const modifiedRequest = new Request(request); +modifiedRequest.headers.set("X-Custom", "value"); +``` + +### "caches is not defined" +Cache API NOT available in Snippets. Use Workers. + +### "Module not found" +Snippets don't support `import`. Use inline code or Workers. + +## Best Practices + +### Performance +- Keep code <10KB (32KB limit) +- Optimize for 5ms CPU +- Clone only when modifying +- Minimize subrequests + +### Security +- Validate all inputs +- Use Web Crypto API for hashing +- Sanitize headers before origin +- Don't log secrets + +### Debugging +```javascript +newResponse.headers.set("X-Debug-Country", request.cf.country); +``` +```bash +curl -H "X-Test: true" https://example.com -v +``` + +## Available APIs + +**✅ Available:** `fetch()`, `Request`, `Response`, `Headers`, `URL`, `crypto.subtle`, `crypto.randomUUID()`, `atob()`/`btoa()`, `JSON` + +**❌ NOT Available:** `caches`, `KV`, `D1`, `R2`, `Durable Objects`, `WebSocket`, `HTMLRewriter`, `import`, Node.js APIs + +## Limits + +| Resource | Limit | +|----------|-------| +| Snippet size | 32KB | +| Execution time | 5ms CPU | +| Subrequests (Pro/Biz) | 2/5 | +| Snippets/zone | 20 | + +## Performance Benchmarks + +| Operation | Time | +|-----------|------| +| Header set | <0.1ms | +| URL parsing | <0.2ms | +| fetch() | 1-3ms | +| SHA-256 | 0.5-1ms | + +**Migrate to Workers when:** >5ms needed, >5 subrequests, need storage (KV/D1/R2), need npm packages, >32KB code diff --git a/plugins/cloudflare/skills/cloudflare/references/snippets/patterns.md b/plugins/cloudflare/skills/cloudflare/references/snippets/patterns.md new file mode 100644 index 0000000..a60c420 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/snippets/patterns.md @@ -0,0 +1,135 @@ +# Snippets Patterns + +## Security Headers + +```javascript +export default { + async fetch(request) { + const response = await fetch(request); + const newResponse = new Response(response.body, response); + newResponse.headers.set("X-Frame-Options", "DENY"); + newResponse.headers.set("X-Content-Type-Options", "nosniff"); + newResponse.headers.delete("X-Powered-By"); + return newResponse; + } +} +``` + +**Rule:** `true` (all requests) + +## Geo-Based Routing + +```javascript +export default { + async fetch(request) { + const country = request.cf.country; + if (["GB", "DE", "FR"].includes(country)) { + const url = new URL(request.url); + url.hostname = url.hostname.replace(".com", ".eu"); + return Response.redirect(url.toString(), 302); + } + return fetch(request); + } +} +``` + +## A/B Testing + +```javascript +export default { + async fetch(request) { + const cookies = request.headers.get("Cookie") || ""; + let variant = cookies.match(/ab_test=([AB])/)?.[1] || (Math.random() < 0.5 ? "A" : "B"); + + const req = new Request(request); + req.headers.set("X-Variant", variant); + const response = await fetch(req); + + if (!cookies.includes("ab_test=")) { + const newResponse = new Response(response.body, response); + newResponse.headers.append("Set-Cookie", `ab_test=${variant}; Path=/; Secure`); + return newResponse; + } + return response; + } +} +``` + +## Bot Detection + +```javascript +export default { + async fetch(request) { + const botScore = request.cf.botManagement?.score; + if (botScore && botScore < 30) return new Response("Denied", { status: 403 }); + return fetch(request); + } +} +``` + +**Requires:** Bot Management plan + +## API Auth Header Injection + +```javascript +export default { + async fetch(request) { + if (new URL(request.url).pathname.startsWith("/api/")) { + const req = new Request(request); + req.headers.set("X-Internal-Auth", "secret_token"); + req.headers.delete("Authorization"); + return fetch(req); + } + return fetch(request); + } +} +``` + +## CORS Headers + +```javascript +export default { + async fetch(request) { + if (request.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE", + "Access-Control-Allow-Headers": "Content-Type, Authorization" + } + }); + } + const response = await fetch(request); + const newResponse = new Response(response.body, response); + newResponse.headers.set("Access-Control-Allow-Origin", "*"); + return newResponse; + } +} +``` + +## Maintenance Mode + +```javascript +export default { + async fetch(request) { + if (request.headers.get("X-Bypass-Token") === "admin") return fetch(request); + return new Response("

Maintenance

", { + status: 503, + headers: { "Content-Type": "text/html", "Retry-After": "3600" } + }); + } +} +``` + +## Pattern Selection + +| Pattern | Complexity | Use Case | +|---------|-----------|----------| +| Security Headers | Low | All sites | +| Geo-Routing | Low | Regional content | +| A/B Testing | Medium | Experiments | +| Bot Detection | Medium | Requires Bot Management | +| API Auth | Low | Backend protection | +| CORS | Low | API endpoints | +| Maintenance | Low | Deployments | diff --git a/plugins/cloudflare/skills/cloudflare/references/spectrum/README.md b/plugins/cloudflare/skills/cloudflare/references/spectrum/README.md new file mode 100644 index 0000000..f78350d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/spectrum/README.md @@ -0,0 +1,52 @@ +# Cloudflare Spectrum Skill Reference + +## Overview + +Cloudflare Spectrum provides security and acceleration for ANY TCP or UDP-based application. It's a global Layer 4 (L4) reverse proxy running on Cloudflare's edge nodes that routes MQTT, email, file transfer, version control, games, and more through Cloudflare to mask origins and protect from DDoS attacks. + +**When to Use Spectrum**: When your protocol isn't HTTP/HTTPS (use Cloudflare proxy for HTTP). Spectrum handles everything else: SSH, gaming, databases, MQTT, SMTP, RDP, custom protocols. + +## Plan Capabilities + +| Capability | Pro/Business | Enterprise | +|------------|--------------|------------| +| TCP protocols | Selected ports only | All ports (1-65535) | +| UDP protocols | Selected ports only | All ports (1-65535) | +| Port ranges | ❌ | ✅ | +| Argo Smart Routing | ✅ | ✅ | +| IP Firewall | ✅ | ✅ | +| Load balancer origins | ✅ | ✅ | + +## Decision Tree + +**What are you trying to do?** + +1. **Create/manage Spectrum app** + - Via Dashboard → See [Cloudflare Dashboard](https://dash.cloudflare.com) + - Via API → See [api.md](api.md) - REST endpoints + - Via SDK → See [api.md](api.md) - TypeScript/Python/Go examples + - Via IaC → See [configuration.md](configuration.md) - Terraform/Pulumi + +2. **Protect specific protocol** + - SSH → See [patterns.md](patterns.md#1-ssh-server-protection) + - Gaming (Minecraft, etc) → See [patterns.md](patterns.md#2-game-server) + - MQTT/IoT → See [patterns.md](patterns.md#3-mqtt-broker) + - SMTP/Email → See [patterns.md](patterns.md#4-smtp-relay) + - Database → See [patterns.md](patterns.md#5-database-proxy) + - RDP → See [patterns.md](patterns.md#6-rdp-remote-desktop) + +3. **Choose origin type** + - Direct IP (single server) → See [configuration.md](configuration.md#direct-ip-origin) + - CNAME (hostname) → See [configuration.md](configuration.md#cname-origin) + - Load balancer (HA/failover) → See [configuration.md](configuration.md#load-balancer-origin) + +## Reading Order + +1. Start with [patterns.md](patterns.md) for your specific protocol +2. Then [configuration.md](configuration.md) for your origin type +3. Check [gotchas.md](gotchas.md) before going to production +4. Use [api.md](api.md) for programmatic access + +## See Also + +- [Cloudflare Docs](https://developers.cloudflare.com/spectrum/) diff --git a/plugins/cloudflare/skills/cloudflare/references/spectrum/api.md b/plugins/cloudflare/skills/cloudflare/references/spectrum/api.md new file mode 100644 index 0000000..645fe2e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/spectrum/api.md @@ -0,0 +1,181 @@ +## REST API Endpoints + +``` +GET /zones/{zone_id}/spectrum/apps # List apps +POST /zones/{zone_id}/spectrum/apps # Create app +GET /zones/{zone_id}/spectrum/apps/{app_id} # Get app +PUT /zones/{zone_id}/spectrum/apps/{app_id} # Update app +DELETE /zones/{zone_id}/spectrum/apps/{app_id} # Delete app + +GET /zones/{zone_id}/spectrum/analytics/aggregate/current +GET /zones/{zone_id}/spectrum/analytics/events/bytime +GET /zones/{zone_id}/spectrum/analytics/events/summary +``` + +## Request/Response Schemas + +### CreateSpectrumAppRequest + +```typescript +interface CreateSpectrumAppRequest { + protocol: string; // "tcp/22", "udp/53" + dns: { + type: "CNAME" | "ADDRESS"; + name: string; // "ssh.example.com" + }; + origin_direct?: string[]; // ["tcp://192.0.2.1:22"] + origin_dns?: { name: string }; // {"name": "origin.example.com"} + origin_port?: number | { start: number; end: number }; + proxy_protocol?: "off" | "v1" | "v2" | "simple"; + ip_firewall?: boolean; + tls?: "off" | "flexible" | "full" | "strict"; + edge_ips?: { + type: "dynamic" | "static"; + connectivity: "all" | "ipv4" | "ipv6"; + }; + traffic_type?: "direct" | "http" | "https"; + argo_smart_routing?: boolean; +} +``` + +### SpectrumApp Response + +```typescript +interface SpectrumApp { + id: string; + protocol: string; + dns: { type: string; name: string }; + origin_direct?: string[]; + origin_dns?: { name: string }; + origin_port?: number | { start: number; end: number }; + proxy_protocol: string; + ip_firewall: boolean; + tls: string; + edge_ips: { type: string; connectivity: string; ips?: string[] }; + argo_smart_routing: boolean; + created_on: string; + modified_on: string; +} +``` + +## TypeScript SDK + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: process.env.CLOUDFLARE_API_TOKEN }); + +// Create +const app = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/22', + dns: { type: 'CNAME', name: 'ssh.example.com' }, + origin_direct: ['tcp://192.0.2.1:22'], + ip_firewall: true, + tls: 'off', +}); + +// List +const apps = await client.spectrum.apps.list({ zone_id: 'your-zone-id' }); + +// Get +const appDetails = await client.spectrum.apps.get({ zone_id: 'your-zone-id', app_id: app.id }); + +// Update +await client.spectrum.apps.update({ zone_id: 'your-zone-id', app_id: app.id, tls: 'full' }); + +// Delete +await client.spectrum.apps.delete({ zone_id: 'your-zone-id', app_id: app.id }); + +// Analytics +const analytics = await client.spectrum.analytics.aggregate({ + zone_id: 'your-zone-id', + metrics: ['bytesIngress', 'bytesEgress'], + since: new Date(Date.now() - 3600000).toISOString(), +}); +``` + +## Python SDK + +```python +from cloudflare import Cloudflare + +client = Cloudflare(api_token="your-api-token") + +# Create +app = client.spectrum.apps.create( + zone_id="your-zone-id", + protocol="tcp/22", + dns={"type": "CNAME", "name": "ssh.example.com"}, + origin_direct=["tcp://192.0.2.1:22"], + ip_firewall=True, + tls="off", +) + +# List +apps = client.spectrum.apps.list(zone_id="your-zone-id") + +# Get +app_details = client.spectrum.apps.get(zone_id="your-zone-id", app_id=app.id) + +# Update +client.spectrum.apps.update(zone_id="your-zone-id", app_id=app.id, tls="full") + +# Delete +client.spectrum.apps.delete(zone_id="your-zone-id", app_id=app.id) + +# Analytics +analytics = client.spectrum.analytics.aggregate( + zone_id="your-zone-id", + metrics=["bytesIngress", "bytesEgress"], + since=datetime.now() - timedelta(hours=1), +) +``` + +## Go SDK + +```go +import "github.com/cloudflare/cloudflare-go" + +api, _ := cloudflare.NewWithAPIToken("your-api-token") + +// Create +app, _ := api.CreateSpectrumApplication(ctx, "zone-id", cloudflare.SpectrumApplication{ + Protocol: "tcp/22", + DNS: cloudflare.SpectrumApplicationDNS{Type: "CNAME", Name: "ssh.example.com"}, + OriginDirect: []string{"tcp://192.0.2.1:22"}, + IPFirewall: true, + ArgoSmartRouting: true, +}) + +// List +apps, _ := api.SpectrumApplications(ctx, "zone-id") + +// Delete +_ = api.DeleteSpectrumApplication(ctx, "zone-id", app.ID) +``` + +## Analytics API + +**Metrics:** +- `bytesIngress` - Bytes received from clients +- `bytesEgress` - Bytes sent to clients +- `count` - Number of connections +- `duration` - Connection duration (seconds) + +**Dimensions:** +- `event` - Connection event type +- `appID` - Spectrum application ID +- `coloName` - Datacenter name +- `ipVersion` - IPv4 or IPv6 + +**Example:** +```bash +curl "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/spectrum/analytics/aggregate/current?metrics=bytesIngress,bytesEgress,count&dimensions=appID" \ + --header "Authorization: Bearer $CLOUDFLARE_API_TOKEN" +``` + +## See Also + +- [configuration.md](configuration.md) - Terraform/Pulumi +- [patterns.md](patterns.md) - Protocol examples diff --git a/plugins/cloudflare/skills/cloudflare/references/spectrum/configuration.md b/plugins/cloudflare/skills/cloudflare/references/spectrum/configuration.md new file mode 100644 index 0000000..81aa72f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/spectrum/configuration.md @@ -0,0 +1,194 @@ +## Origin Types + +### Direct IP Origin + +Use when origin is a single server with static IP. + +**TypeScript SDK:** +```typescript +const app = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/22', + dns: { type: 'CNAME', name: 'ssh.example.com' }, + origin_direct: ['tcp://192.0.2.1:22'], + ip_firewall: true, + tls: 'off', +}); +``` + +**Terraform:** +```hcl +resource "cloudflare_spectrum_application" "ssh" { + zone_id = var.zone_id + protocol = "tcp/22" + + dns { + type = "CNAME" + name = "ssh.example.com" + } + + origin_direct = ["tcp://192.0.2.1:22"] + ip_firewall = true + tls = "off" + argo_smart_routing = true +} +``` + +### CNAME Origin + +Use when origin is a hostname (not static IP). Spectrum resolves DNS dynamically. + +**TypeScript SDK:** +```typescript +const app = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/3306', + dns: { type: 'CNAME', name: 'db.example.com' }, + origin_dns: { name: 'db-primary.internal.example.com' }, + origin_port: 3306, + tls: 'full', +}); +``` + +**Terraform:** +```hcl +resource "cloudflare_spectrum_application" "database" { + zone_id = var.zone_id + protocol = "tcp/3306" + + dns { + type = "CNAME" + name = "db.example.com" + } + + origin_dns { + name = "db-primary.internal.example.com" + } + + origin_port = 3306 + tls = "full" + argo_smart_routing = true +} +``` + +### Load Balancer Origin + +Use for high availability and failover. + +**Terraform:** +```hcl +resource "cloudflare_load_balancer" "game_lb" { + zone_id = var.zone_id + name = "game-lb.example.com" + default_pool_ids = [cloudflare_load_balancer_pool.game_pool.id] +} + +resource "cloudflare_load_balancer_pool" "game_pool" { + name = "game-primary" + origins { name = "game-1"; address = "192.0.2.1" } + monitor = cloudflare_load_balancer_monitor.tcp_monitor.id +} + +resource "cloudflare_load_balancer_monitor" "tcp_monitor" { + type = "tcp"; port = 25565; interval = 60; timeout = 5 +} + +resource "cloudflare_spectrum_application" "game" { + zone_id = var.zone_id + protocol = "tcp/25565" + dns { type = "CNAME"; name = "game.example.com" } + origin_dns { name = cloudflare_load_balancer.game_lb.name } + origin_port = 25565 +} +``` + +## TLS Configuration + +| Mode | Description | Use Case | Origin Cert | +|------|-------------|----------|-------------| +| `off` | No TLS | Non-encrypted (SSH, gaming) | No | +| `flexible` | TLS client→CF, plain CF→origin | Testing | No | +| `full` | TLS end-to-end, self-signed OK | Production | Yes (any) | +| `strict` | Full + valid cert verification | Max security | Yes (CA) | + +**Example:** +```typescript +const app = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/3306', + dns: { type: 'CNAME', name: 'db.example.com' }, + origin_direct: ['tcp://192.0.2.1:3306'], + tls: 'strict', // Validates origin certificate +}); +``` + +## Proxy Protocol + +Forwards real client IP to origin. Origin must support parsing. + +| Version | Protocol | Use Case | +|---------|----------|----------| +| `off` | - | Origin doesn't need client IP | +| `v1` | TCP | Most TCP apps (SSH, databases) | +| `v2` | TCP | High-performance TCP | +| `simple` | UDP | UDP applications | + +**Compatibility:** +- **v1**: HAProxy, nginx, SSH, most databases +- **v2**: HAProxy 1.5+, nginx 1.11+ +- **simple**: Cloudflare-specific UDP format + +**Enable:** +```typescript +const app = await client.spectrum.apps.create({ + // ... + proxy_protocol: 'v1', // Origin must parse PROXY header +}); +``` + +**Origin Config (nginx):** +```nginx +stream { + server { + listen 22 proxy_protocol; + proxy_pass backend:22; + } +} +``` + +## IP Access Rules + +Enable `ip_firewall: true` then configure zone-level firewall rules. + +```typescript +const app = await client.spectrum.apps.create({ + // ... + ip_firewall: true, // Applies zone firewall rules +}); +``` + +## Port Ranges (Enterprise Only) + +```hcl +resource "cloudflare_spectrum_application" "game_cluster" { + zone_id = var.zone_id + protocol = "tcp/25565-25575" + + dns { + type = "CNAME" + name = "games.example.com" + } + + origin_direct = ["tcp://192.0.2.1"] + + origin_port { + start = 25565 + end = 25575 + } +} +``` + +## See Also + +- [patterns.md](patterns.md) - Protocol-specific examples +- [api.md](api.md) - REST/SDK reference diff --git a/plugins/cloudflare/skills/cloudflare/references/spectrum/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/spectrum/gotchas.md new file mode 100644 index 0000000..ef31a36 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/spectrum/gotchas.md @@ -0,0 +1,145 @@ +## Common Issues + +### Connection Timeouts + +**Problem:** Connections fail or timeout +**Cause:** Origin firewall blocking Cloudflare IPs, origin service not running, incorrect DNS +**Solution:** +1. Verify origin firewall allows Cloudflare IP ranges +2. Check origin service running on correct port +3. Ensure DNS record is CNAME (not A/AAAA) +4. Verify origin IP/hostname is correct + +```bash +# Test connectivity +nc -zv app.example.com 22 +dig app.example.com +``` + +### Client IP Showing Cloudflare IP + +**Problem:** Origin logs show Cloudflare IPs not real client IPs +**Cause:** Proxy Protocol not enabled or origin not configured +**Solution:** +```typescript +// Enable in Spectrum app +const app = await client.spectrum.apps.create({ + // ... + proxy_protocol: 'v1', // TCP: v1/v2; UDP: simple +}); +``` + +**Origin config:** +- **nginx**: `listen 22 proxy_protocol;` +- **HAProxy**: `bind :22 accept-proxy` + +### TLS Errors + +**Problem:** TLS handshake failures, 525 errors +**Cause:** TLS mode mismatch + +| Error | TLS Mode | Problem | Solution | +|-------|----------|---------|----------| +| Connection refused | `full`/`strict` | Origin not TLS | Use `tls: "off"` or enable TLS | +| 525 cert invalid | `strict` | Self-signed cert | Use `tls: "full"` or valid cert | +| Handshake timeout | `flexible` | Origin expects TLS | Use `tls: "full"` | + +**Debug:** +```bash +openssl s_client -connect app.example.com:443 -showcerts +``` + +### SMTP Reverse DNS + +**Problem:** Email servers reject SMTP via Spectrum +**Cause:** Spectrum IPs lack PTR (reverse DNS) records +**Impact:** Many mail servers require valid rDNS for anti-spam + +**Solution:** +- Outbound SMTP: NOT recommended through Spectrum +- Inbound SMTP: Use Cloudflare Email Routing +- Internal relay: Whitelist Spectrum IPs on destination + +### Proxy Protocol Compatibility + +**Problem:** Connection works but app behaves incorrectly +**Cause:** Origin doesn't support Proxy Protocol + +**Solution:** +1. Verify origin supports version (v1: widely supported, v2: HAProxy 1.5+/nginx 1.11+) +2. Test with `proxy_protocol: 'off'` first +3. Configure origin to parse headers + +**nginx TCP:** +```nginx +stream { + server { + listen 22 proxy_protocol; + proxy_pass backend:22; + } +} +``` + +**HAProxy:** +``` +frontend ft_ssh + bind :22 accept-proxy +``` + +### Analytics Data Retention + +**Problem:** Historical data not available +**Cause:** Retention varies by plan + +| Plan | Real-time | Historical | +|------|-----------|------------| +| Pro | Last hour | ❌ | +| Business | Last hour | Limited | +| Enterprise | Last hour | 90+ days | + +**Solution:** Query within retention window or export to external system + +### Enterprise-Only Features + +**Problem:** Feature unavailable/errors +**Cause:** Requires Enterprise plan + +**Enterprise-only:** +- Port ranges (`tcp/25565-25575`) +- All TCP/UDP ports (Pro/Business: selected only) +- Extended analytics retention +- Advanced load balancing + +### IPv6 Considerations + +**Problem:** IPv6 clients can't connect or origin doesn't support IPv6 +**Solution:** Configure `edge_ips.connectivity` + +```typescript +const app = await client.spectrum.apps.create({ + // ... + edge_ips: { + type: 'dynamic', + connectivity: 'ipv4', // Options: 'all', 'ipv4', 'ipv6' + }, +}); +``` + +**Options:** +- `all`: Dual-stack (default, requires origin support both) +- `ipv4`: IPv4 only (use if origin lacks IPv6) +- `ipv6`: IPv6 only (rare) + +## Limits + +| Resource | Pro/Business | Enterprise | +|----------|--------------|------------| +| Max apps | ~10-15 | 100+ | +| Protocols | Selected | All TCP/UDP | +| Port ranges | ❌ | ✅ | +| Analytics | ~1 hour | 90+ days | + +## See Also + +- [patterns.md](patterns.md) - Protocol examples +- [configuration.md](configuration.md) - TLS/Proxy setup diff --git a/plugins/cloudflare/skills/cloudflare/references/spectrum/patterns.md b/plugins/cloudflare/skills/cloudflare/references/spectrum/patterns.md new file mode 100644 index 0000000..4032486 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/spectrum/patterns.md @@ -0,0 +1,196 @@ +## Common Use Cases + +### 1. SSH Server Protection + +**Terraform:** +```hcl +resource "cloudflare_spectrum_application" "ssh" { + zone_id = var.zone_id + protocol = "tcp/22" + + dns { + type = "CNAME" + name = "ssh.example.com" + } + + origin_direct = ["tcp://10.0.1.5:22"] + ip_firewall = true + argo_smart_routing = true +} +``` + +**Benefits:** Hide origin IP, DDoS protection, IP firewall, Argo reduces latency + +### 2. Game Server + +**TypeScript (Minecraft):** +```typescript +const app = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/25565', + dns: { type: 'CNAME', name: 'mc.example.com' }, + origin_direct: ['tcp://192.168.1.10:25565'], + proxy_protocol: 'v1', // Preserves player IPs + argo_smart_routing: true, +}); +``` + +**Benefits:** DDoS protection, hide origin IP, Proxy Protocol for player IPs/bans, Argo reduces latency + +### 3. MQTT Broker + +IoT device communication. + +**TypeScript:** +```typescript +const mqttApp = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/8883', // Use 1883 for plain MQTT + dns: { type: 'CNAME', name: 'mqtt.example.com' }, + origin_direct: ['tcp://mqtt-broker.internal:8883'], + tls: 'full', // Use 'off' for plain MQTT +}); +``` + +**Benefits:** DDoS protection, hide broker IP, TLS termination at edge + +### 4. SMTP Relay + +Email submission (port 587). **WARNING**: See [gotchas.md](gotchas.md#smtp-reverse-dns) + +**Terraform:** +```hcl +resource "cloudflare_spectrum_application" "smtp" { + zone_id = var.zone_id + protocol = "tcp/587" + + dns { + type = "CNAME" + name = "smtp.example.com" + } + + origin_direct = ["tcp://mail-server.internal:587"] + tls = "full" # STARTTLS support +} +``` + +**Limitations:** +- Spectrum IPs lack reverse DNS (PTR records) +- Many mail servers reject without valid rDNS +- Best for internal/trusted relay only + +### 5. Database Proxy + +MySQL/PostgreSQL. **Use with caution** - security critical. + +**PostgreSQL:** +```typescript +const postgresApp = await client.spectrum.apps.create({ + zone_id: 'your-zone-id', + protocol: 'tcp/5432', + dns: { type: 'CNAME', name: 'postgres.example.com' }, + origin_dns: { name: 'db-primary.internal.example.com' }, + origin_port: 5432, + tls: 'strict', // REQUIRED + ip_firewall: true, // REQUIRED +}); +``` + +**MySQL:** +```hcl +resource "cloudflare_spectrum_application" "mysql" { + zone_id = var.zone_id + protocol = "tcp/3306" + + dns { + type = "CNAME" + name = "mysql.example.com" + } + + origin_dns { + name = "mysql-primary.internal.example.com" + } + + origin_port = 3306 + tls = "strict" + ip_firewall = true +} +``` + +**Security:** +- ALWAYS use `tls: "strict"` +- ALWAYS use `ip_firewall: true` +- Restrict to known IPs via zone firewall +- Use strong DB authentication +- Consider VPN or Cloudflare Access instead + +### 6. RDP (Remote Desktop) + +**Requires IP firewall.** + +**Terraform:** +```hcl +resource "cloudflare_spectrum_application" "rdp" { + zone_id = var.zone_id + protocol = "tcp/3389" + + dns { + type = "CNAME" + name = "rdp.example.com" + } + + origin_direct = ["tcp://windows-server.internal:3389"] + tls = "off" # RDP has own encryption + ip_firewall = true # REQUIRED +} +``` + +**Security:** ALWAYS `ip_firewall: true`, whitelist admin IPs, RDP is DDoS/brute-force target + +### 7. Multi-Origin Failover + +High availability with load balancer. + +**Terraform:** +```hcl +resource "cloudflare_load_balancer" "database_lb" { + zone_id = var.zone_id + name = "db-lb.example.com" + default_pool_ids = [cloudflare_load_balancer_pool.db_primary.id] + fallback_pool_id = cloudflare_load_balancer_pool.db_secondary.id +} + +resource "cloudflare_load_balancer_pool" "db_primary" { + name = "db-primary-pool" + origins { name = "db-1"; address = "192.0.2.1" } + monitor = cloudflare_load_balancer_monitor.postgres_monitor.id +} + +resource "cloudflare_load_balancer_pool" "db_secondary" { + name = "db-secondary-pool" + origins { name = "db-2"; address = "192.0.2.2" } + monitor = cloudflare_load_balancer_monitor.postgres_monitor.id +} + +resource "cloudflare_load_balancer_monitor" "postgres_monitor" { + type = "tcp"; port = 5432; interval = 30; timeout = 5 +} + +resource "cloudflare_spectrum_application" "postgres_ha" { + zone_id = var.zone_id + protocol = "tcp/5432" + dns { type = "CNAME"; name = "postgres.example.com" } + origin_dns { name = cloudflare_load_balancer.database_lb.name } + origin_port = 5432 + tls = "strict" + ip_firewall = true +} +``` + +**Benefits:** Automatic failover, health monitoring, traffic distribution, zero-downtime deployments + +## See Also + +- [configuration.md](configuration.md) - Origin type setup +- [gotchas.md](gotchas.md) - Protocol limitations +- [api.md](api.md) - SDK reference diff --git a/plugins/cloudflare/skills/cloudflare/references/static-assets/README.md b/plugins/cloudflare/skills/cloudflare/references/static-assets/README.md new file mode 100644 index 0000000..b2ba96a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/static-assets/README.md @@ -0,0 +1,65 @@ +# Cloudflare Static Assets Skill Reference + +Expert guidance for deploying and configuring static assets with Cloudflare Workers. This skill covers configuration patterns, routing architectures, asset binding usage, and best practices for SPAs, SSG sites, and full-stack applications. + +## Quick Start + +```jsonc +// wrangler.jsonc +{ + "name": "my-app", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", + "assets": { + "directory": "./dist" + } +} +``` + +```typescript +// src/index.ts +export default { + async fetch(request: Request, env: Env): Promise { + return env.ASSETS.fetch(request); + } +}; +``` + +Deploy: `wrangler deploy` + +## When to Use Workers Static Assets vs Pages + +| Factor | Workers Static Assets | Cloudflare Pages | +|--------|----------------------|------------------| +| **Use case** | Hybrid apps (static + dynamic API) | Static sites, SSG | +| **Worker control** | Full control over routing | Limited (Functions) | +| **Configuration** | Code-first, flexible | Git-based, opinionated | +| **Dynamic routing** | Worker-first patterns | Functions (_functions/) | +| **Best for** | Full-stack apps, SPAs with APIs | Jamstack, static docs | + +**Decision tree:** + +- Need custom routing logic? → Workers Static Assets +- Pure static site or SSG? → Pages +- API routes + SPA? → Workers Static Assets +- Framework (Next, Nuxt, Remix)? → Pages + +## Reading Order + +1. **configuration.md** - Setup, wrangler.jsonc options, routing patterns +2. **api.md** - ASSETS binding API, request/response handling +3. **patterns.md** - Common patterns (SPA, API routes, auth, A/B testing) +4. **gotchas.md** - Limits, errors, performance tips + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, configuration +- **[api.md](api.md)** - API endpoints, methods, interfaces +- **[patterns.md](patterns.md)** - Common patterns, use cases, examples +- **[gotchas.md](gotchas.md)** - Troubleshooting, best practices, limitations + +## See Also + +- [Cloudflare Workers Docs](https://developers.cloudflare.com/workers/) +- [Static Assets Docs](https://developers.cloudflare.com/workers/static-assets/) +- [Cloudflare Pages](https://developers.cloudflare.com/pages/) diff --git a/plugins/cloudflare/skills/cloudflare/references/static-assets/api.md b/plugins/cloudflare/skills/cloudflare/references/static-assets/api.md new file mode 100644 index 0000000..08bb568 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/static-assets/api.md @@ -0,0 +1,199 @@ +# API Reference + +## ASSETS Binding + +The `ASSETS` binding provides access to static assets via the `Fetcher` interface. + +### Type Definition + +```typescript +interface Env { + ASSETS: Fetcher; +} + +interface Fetcher { + fetch(input: RequestInfo | URL, init?: RequestInit): Promise; +} +``` + +### Method Signatures + +```typescript +// 1. Forward entire request +await env.ASSETS.fetch(request); + +// 2. String path (hostname ignored, only path matters) +await env.ASSETS.fetch("https://any-host/path/to/asset.png"); + +// 3. URL object +await env.ASSETS.fetch(new URL("/index.html", request.url)); + +// 4. Constructed Request object +await env.ASSETS.fetch(new Request(new URL("/logo.png", request.url), { + method: "GET", + headers: request.headers +})); +``` + +**Key behaviors:** + +- Host/origin is ignored for string/URL inputs (only path is used) +- Method must be GET (others return 405) +- Request headers pass through (affects response) +- Returns standard `Response` object + +## Request Handling + +### Path Resolution + +```typescript +// All resolve to same asset: +env.ASSETS.fetch("https://example.com/logo.png") +env.ASSETS.fetch("https://ignored.host/logo.png") +env.ASSETS.fetch("/logo.png") +``` + +Assets are resolved relative to configured `assets.directory`. + +### Headers + +Request headers that affect response: + +| Header | Effect | +|--------|--------| +| `Accept-Encoding` | Controls compression (gzip, brotli) | +| `Range` | Enables partial content (206 responses) | +| `If-None-Match` | Conditional request via ETag | +| `If-Modified-Since` | Conditional request via modification date | + +Custom headers pass through but don't affect asset serving. + +### Method Support + +| Method | Supported | Response | +|--------|-----------|----------| +| `GET` | ✅ Yes | Asset content | +| `HEAD` | ✅ Yes | Headers only, no body | +| `POST`, `PUT`, etc. | ❌ No | 405 Method Not Allowed | + +## Response Behavior + +### Content-Type Inference + +Automatically set based on file extension: + +| Extension | Content-Type | +|-----------|--------------| +| `.html` | `text/html; charset=utf-8` | +| `.css` | `text/css` | +| `.js` | `application/javascript` | +| `.json` | `application/json` | +| `.png` | `image/png` | +| `.jpg`, `.jpeg` | `image/jpeg` | +| `.svg` | `image/svg+xml` | +| `.woff2` | `font/woff2` | + +### Default Headers + +Responses include: + +``` +Content-Type: +ETag: "" +Cache-Control: public, max-age=3600 +Content-Encoding: br (if supported and beneficial) +``` + +**Cache-Control defaults:** + +- 1 hour (`max-age=3600`) for most assets +- Override via Worker response transformation (see patterns.md:27-35) + +### Compression + +Automatic compression based on `Accept-Encoding`: + +- **Brotli** (`br`): Preferred, best compression +- **Gzip** (`gzip`): Fallback +- **None**: If client doesn't support or asset too small + +### ETag Generation + +ETags are content-based hashes: + +``` +ETag: "a3b2c1d4e5f6..." +``` + +Used for conditional requests (`If-None-Match`). Returns `304 Not Modified` if match. + +## Error Responses + +| Status | Condition | Behavior | +|--------|-----------|----------| +| `404` | Asset not found | Body depends on `not_found_handling` config | +| `405` | Non-GET/HEAD method | `{ "error": "Method not allowed" }` | +| `416` | Invalid Range header | Range not satisfiable | + +### 404 Handling + +Depends on configuration (see configuration.md:45-52): + +```typescript +// not_found_handling: "single-page-application" +// Returns /index.html with 200 status + +// not_found_handling: "404-page" +// Returns /404.html if exists, else 404 response + +// not_found_handling: "none" +// Returns 404 response +``` + +## Advanced Usage + +### Modifying Responses + +```typescript +const response = await env.ASSETS.fetch(request); + +// Clone and modify +return new Response(response.body, { + status: response.status, + headers: { + ...Object.fromEntries(response.headers), + 'Cache-Control': 'public, max-age=31536000', + 'X-Custom': 'value' + } +}); +``` + +See patterns.md:27-35 for full example. + +### Error Handling + +```typescript +const response = await env.ASSETS.fetch(request); + +if (!response.ok) { + // Asset not found or error + return new Response('Custom error page', { status: 404 }); +} + +return response; +``` + +### Conditional Serving + +```typescript +const url = new URL(request.url); + +// Serve different assets based on conditions +if (url.pathname === '/') { + return env.ASSETS.fetch('/index.html'); +} + +return env.ASSETS.fetch(request); +``` + +See patterns.md for complete patterns. diff --git a/plugins/cloudflare/skills/cloudflare/references/static-assets/configuration.md b/plugins/cloudflare/skills/cloudflare/references/static-assets/configuration.md new file mode 100644 index 0000000..2902698 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/static-assets/configuration.md @@ -0,0 +1,186 @@ +## Configuration + +### Basic Setup + +Minimal configuration requires only `assets.directory`: + +```jsonc +{ + "name": "my-worker", + "compatibility_date": "2025-01-01", // Use current date for new projects + "assets": { + "directory": "./dist" + } +} +``` + +### Full Configuration Options + +```jsonc +{ + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", + "assets": { + "directory": "./dist", + "binding": "ASSETS", + "not_found_handling": "single-page-application", + "html_handling": "auto-trailing-slash", + "run_worker_first": ["/api/*", "!/api/docs/*"] + } +} +``` + +**Configuration keys:** + +- `directory` (string, required): Path to assets folder (e.g. `./dist`, `./public`, `./build`) +- `binding` (string, optional): Name to access assets in Worker code (e.g. `env.ASSETS`). Default: `"ASSETS"` +- `not_found_handling` (string, optional): Behavior when asset not found + - `"single-page-application"`: Serve `/index.html` for non-asset paths (default for SPAs) + - `"404-page"`: Serve `/404.html` if present, otherwise 404 + - `"none"`: Return 404 for missing assets +- `html_handling` (string, optional): URL trailing slash behavior +- `run_worker_first` (boolean | string[], optional): Routes that invoke Worker before checking assets + +### not_found_handling Modes + +| Mode | Behavior | Use Case | +|------|----------|----------| +| `"single-page-application"` | Serve `/index.html` for non-asset requests | React, Vue, Angular SPAs | +| `"404-page"` | Serve `/404.html` if exists, else 404 | Static sites with custom error page | +| `"none"` | Return 404 for missing assets | API-first or custom routing | + +### html_handling Modes + +Controls trailing slash behavior for HTML files: + +| Mode | `/page` | `/page/` | Use Case | +|------|---------|----------|----------| +| `"auto-trailing-slash"` | Redirect to `/page/` if `/page/index.html` exists | Serve `/page/index.html` | Default, SEO-friendly | +| `"force-trailing-slash"` | Always redirect to `/page/` | Serve if exists | Consistent trailing slashes | +| `"drop-trailing-slash"` | Serve if exists | Redirect to `/page` | Cleaner URLs | +| `"none"` | No modification | No modification | Custom routing logic | + +**Default:** `"auto-trailing-slash"` + +### run_worker_first Configuration + +Controls which requests invoke Worker before checking assets. + +**Boolean syntax:** + +```jsonc +{ + "assets": { + "run_worker_first": true // ALL requests invoke Worker + } +} +``` + +**Array syntax (recommended):** + +```jsonc +{ + "assets": { + "run_worker_first": [ + "/api/*", // Positive pattern: match API routes + "/admin/*", // Match admin routes + "!/admin/assets/*" // Negative pattern: exclude admin assets + ] + } +} +``` + +**Pattern rules:** + +- Glob patterns: `*` (any chars), `**` (any path segments) +- Negative patterns: Prefix with `!` to exclude +- Precedence: Negative patterns override positive patterns +- Default: `false` (assets served directly) + +**Decision guidance:** + +- Use `true` for API-first apps (few static assets) +- Use array patterns for hybrid apps (APIs + static assets) +- Use `false` for static-first sites (minimal dynamic routes) + +### .assetsignore File + +Exclude files from upload using `.assetsignore` (same syntax as `.gitignore`): + +``` +# .assetsignore +_worker.js +*.map +*.md +node_modules/ +.git/ +``` + +**Common patterns:** + +- `_worker.js` - Exclude Worker code from assets +- `*.map` - Exclude source maps +- `*.md` - Exclude markdown files +- Development artifacts + +### Vite Plugin Integration + +For Vite-based projects, use `@cloudflare/vite-plugin`: + +```typescript +// vite.config.ts +import { defineConfig } from 'vite'; +import { cloudflare } from '@cloudflare/vite-plugin'; + +export default defineConfig({ + plugins: [ + cloudflare({ + assets: { + directory: './dist', + binding: 'ASSETS' + } + }) + ] +}); +``` + +**Features:** + +- Automatic asset detection during dev +- Hot module replacement for assets +- Production build integration +- Requires: Wrangler 4.0.0+, `@cloudflare/vite-plugin` 1.0.0+ + +### Key Compatibility Dates + +| Date | Feature | Impact | +|------|---------|--------| +| `2025-04-01` | Navigation request optimization | SPAs skip Worker for navigation, reducing costs | + +Use current date for new projects. See [Compatibility Dates](https://developers.cloudflare.com/workers/configuration/compatibility-dates/) for full list. + +### Environment-Specific Configuration + +Use `wrangler.jsonc` environments for different configs: + +```jsonc +{ + "name": "my-worker", + "assets": { "directory": "./dist" }, + "env": { + "staging": { + "assets": { + "not_found_handling": "404-page" + } + }, + "production": { + "assets": { + "not_found_handling": "single-page-application" + } + } + } +} +``` + +Deploy with: `wrangler deploy --env staging` diff --git a/plugins/cloudflare/skills/cloudflare/references/static-assets/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/static-assets/gotchas.md new file mode 100644 index 0000000..2577f17 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/static-assets/gotchas.md @@ -0,0 +1,162 @@ +## Best Practices + +### 1. Use Selective Worker-First Routing + +Instead of `run_worker_first = true`, use array patterns: + +```jsonc +{ + "assets": { + "run_worker_first": [ + "/api/*", // API routes + "/admin/*", // Admin area + "!/admin/assets/*" // Except admin assets + ] + } +} +``` + +**Benefits:** +- Reduces Worker invocations +- Lowers costs +- Improves asset delivery performance + +### 2. Leverage Navigation Request Optimization + +For SPAs, use `compatibility_date = "2025-04-01"` or later: + +```jsonc +{ + "compatibility_date": "2025-04-01", + "assets": { + "not_found_handling": "single-page-application" + } +} +``` + +Navigation requests skip Worker invocation, reducing costs. + +### 3. Type Safety with Bindings + +Always type your environment: + +```typescript +interface Env { + ASSETS: Fetcher; +} +``` + +## Common Errors + +### "Asset not found" + +**Cause:** Asset not in assets directory, wrong path, or assets not deployed +**Solution:** Verify asset exists, check path case-sensitivity, redeploy if needed + +### "Worker not invoked for asset" + +**Cause:** Asset served directly, `run_worker_first` not configured +**Solution:** Configure `run_worker_first` patterns to include asset routes (see configuration.md:66-106) + +### "429 Too Many Requests on free tier" + +**Cause:** `run_worker_first` patterns invoke Worker for many requests, hitting free tier limits (100k req/day) +**Solution:** Use more selective patterns with negative exclusions, or upgrade to paid plan + +### "Smart Placement increases latency" + +**Cause:** `run_worker_first=true` + Smart Placement routes all requests through single smart-placed location +**Solution:** Use selective patterns (array syntax) or disable Smart Placement for asset-heavy apps + +### "CF-Cache-Status header unreliable" + +**Cause:** Header is probabilistically added for privacy reasons +**Solution:** Don't rely on `CF-Cache-Status` for critical routing logic. Use other signals (ETag, age). + +### "JWT expired during deployment" + +**Cause:** Large asset deployments exceed JWT token lifetime +**Solution:** Update to Wrangler 4.34.0+ (automatic token refresh), or reduce asset count + +### "Cannot use 'assets' with 'site'" + +**Cause:** Legacy `site` config conflicts with new `assets` config +**Solution:** Migrate from `site` to `assets` (see configuration.md). Remove `site` key from wrangler.jsonc. + +### "Assets not updating after deployment" + +**Cause:** Browser or CDN cache serving old assets +**Solution:** +- Hard refresh browser (Cmd+Shift+R / Ctrl+F5) +- Use cache-busting (hashed filenames) +- Verify deployment completed: `wrangler tail` + +## Limits + +| Resource/Limit | Free | Paid | Notes | +|----------------|------|------|-------| +| Max asset size | 25 MiB | 25 MiB | Per file | +| Total assets | 20,000 | **100,000** | Requires Wrangler 4.34.0+ (Sep 2025) | +| Worker invocations | 100k/day | 10M/month | Optimize with `run_worker_first` patterns | +| Asset storage | Unlimited | Unlimited | Included | + +### Version Requirements + +| Feature | Minimum Wrangler Version | +|---------|--------------------------| +| 100k file limit (paid) | 4.34.0 | +| Vite plugin | 4.0.0 + @cloudflare/vite-plugin 1.0.0 | +| Navigation optimization | 4.0.0 + compatibility_date: "2025-04-01" | + +## Performance Tips + +### 1. Use Hashed Filenames + +Enable long-term caching with content-hashed filenames: + +``` +app.a3b2c1d4.js +styles.e5f6g7h8.css +``` + +Most bundlers (Vite, Webpack, Parcel) do this automatically. + +### 2. Minimize Worker Invocations + +Serve assets directly when possible: + +```jsonc +{ + "assets": { + // Only invoke Worker for dynamic routes + "run_worker_first": ["/api/*", "/auth/*"] + } +} +``` + +### 3. Leverage Browser Cache + +Set appropriate `Cache-Control` headers: + +```typescript +// Versioned assets +'Cache-Control': 'public, max-age=31536000, immutable' + +// HTML (revalidate often) +'Cache-Control': 'public, max-age=0, must-revalidate' +``` + +See patterns.md:169-189 for implementation. + +### 4. Use .assetsignore + +Reduce upload time by excluding unnecessary files: + +``` +*.map +*.md +.DS_Store +node_modules/ +``` + +See configuration.md:107-126 for details. diff --git a/plugins/cloudflare/skills/cloudflare/references/static-assets/patterns.md b/plugins/cloudflare/skills/cloudflare/references/static-assets/patterns.md new file mode 100644 index 0000000..11ddda2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/static-assets/patterns.md @@ -0,0 +1,189 @@ +### Common Patterns + +**1. Forward request to assets:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + return env.ASSETS.fetch(request); + } +}; +``` + +**2. Fetch specific asset by path:** + +```typescript +const response = await env.ASSETS.fetch("https://assets.local/logo.png"); +``` + +**3. Modify request before fetching asset:** + +```typescript +const url = new URL(request.url); +url.pathname = "/index.html"; +return env.ASSETS.fetch(new Request(url, request)); +``` + +**4. Transform asset response:** + +```typescript +const response = await env.ASSETS.fetch(request); +const modifiedResponse = new Response(response.body, response); +modifiedResponse.headers.set("X-Custom-Header", "value"); +modifiedResponse.headers.set("Cache-Control", "public, max-age=3600"); +return modifiedResponse; +``` + +**5. Conditional asset serving:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname === '/') { + return env.ASSETS.fetch('/index.html'); + } + return env.ASSETS.fetch(request); + } +}; +``` + +**6. SPA with API routes:** + +Most common full-stack pattern - static SPA with backend API: + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname.startsWith('/api/')) { + return handleAPI(request, env); + } + return env.ASSETS.fetch(request); + } +}; + +async function handleAPI(request: Request, env: Env): Promise { + return new Response(JSON.stringify({ status: 'ok' }), { + headers: { 'Content-Type': 'application/json' } + }); +} +``` + +**Config:** Set `run_worker_first: ["/api/*"]` (see configuration.md:66-106) + +**7. Auth gating for protected assets:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname.startsWith('/admin/')) { + const session = await validateSession(request, env); + if (!session) { + return Response.redirect('/login', 302); + } + } + return env.ASSETS.fetch(request); + } +}; +``` + +**Config:** Set `run_worker_first: ["/admin/*"]` + +**8. Custom headers for security:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const response = await env.ASSETS.fetch(request); + const secureResponse = new Response(response.body, response); + secureResponse.headers.set('X-Frame-Options', 'DENY'); + secureResponse.headers.set('X-Content-Type-Options', 'nosniff'); + secureResponse.headers.set('Content-Security-Policy', "default-src 'self'"); + return secureResponse; + } +}; +``` + +**9. A/B testing via cookies:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const cookies = request.headers.get('Cookie') || ''; + const variant = cookies.includes('variant=b') ? 'b' : 'a'; + const url = new URL(request.url); + if (url.pathname === '/') { + return env.ASSETS.fetch(`/index-${variant}.html`); + } + return env.ASSETS.fetch(request); + } +}; +``` + +**10. Locale-based routing:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const locale = request.headers.get('Accept-Language')?.split(',')[0] || 'en'; + const url = new URL(request.url); + if (url.pathname === '/') { + return env.ASSETS.fetch(`/${locale}/index.html`); + } + if (!url.pathname.startsWith(`/${locale}/`)) { + url.pathname = `/${locale}${url.pathname}`; + } + return env.ASSETS.fetch(url); + } +}; +``` + +**11. OAuth callback handling:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname === '/auth/callback') { + const code = url.searchParams.get('code'); + if (code) { + const session = await exchangeCode(code, env); + return new Response(null, { + status: 302, + headers: { + 'Location': '/', + 'Set-Cookie': `session=${session}; HttpOnly; Secure; SameSite=Lax` + } + }); + } + } + return env.ASSETS.fetch(request); + } +}; +``` + +**Config:** Set `run_worker_first: ["/auth/*"]` + +**12. Cache control override:** + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const response = await env.ASSETS.fetch(request); + const url = new URL(request.url); + // Immutable assets (hashed filenames) + if (/\.[a-f0-9]{8,}\.(js|css|png|jpg)$/.test(url.pathname)) { + return new Response(response.body, { + ...response, + headers: { + ...Object.fromEntries(response.headers), + 'Cache-Control': 'public, max-age=31536000, immutable' + } + }); + } + return response; + } +}; +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/stream/README.md b/plugins/cloudflare/skills/cloudflare/references/stream/README.md new file mode 100644 index 0000000..be251e3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/stream/README.md @@ -0,0 +1,114 @@ +# Cloudflare Stream + +Serverless live and on-demand video streaming platform with one API. + +## Overview + +Cloudflare Stream provides video upload, storage, encoding, and delivery without managing infrastructure. Runs on Cloudflare's global network. + +### Key Features +- **On-demand video**: Upload, encode, store, deliver +- **Live streaming**: RTMPS/SRT ingestion with ABR +- **Direct creator uploads**: End users upload without API keys +- **Signed URLs**: Token-based access control +- **Analytics**: Server-side metrics via GraphQL +- **Webhooks**: Processing notifications +- **Captions**: Upload or AI-generate subtitles +- **Watermarks**: Apply branding to videos +- **Downloads**: Enable MP4 offline viewing + +## Core Concepts + +### Video Upload Methods +1. **API Upload (TUS protocol)**: Direct server upload +2. **Upload from URL**: Import from external source +3. **Direct Creator Uploads**: User-generated content (recommended) + +### Playback Options +1. **Stream Player (iframe)**: Built-in, optimized player +2. **Custom Player (HLS/DASH)**: Video.js, HLS.js integration +3. **Thumbnails**: Static or animated previews + +### Access Control +- **Public**: No restrictions +- **requireSignedURLs**: Token-based access +- **allowedOrigins**: Domain restrictions +- **Access Rules**: Geo/IP restrictions in tokens + +### Live Streaming +- RTMPS/SRT ingest from OBS, FFmpeg +- Automatic recording to on-demand +- Simulcast to YouTube, Twitch, etc. +- WebRTC support for browser streaming + +## Quick Start + +**Upload video via API** +```bash +curl -X POST \ + "https://api.cloudflare.com/client/v4/accounts/{account_id}/stream/copy" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"url": "https://example.com/video.mp4"}' +``` + +**Embed player** +```html + +``` + +**Create live input** +```bash +curl -X POST \ + "https://api.cloudflare.com/client/v4/accounts/{account_id}/stream/live_inputs" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"recording": {"mode": "automatic"}}' +``` + +## Limits + +- Max file size: 30 GB +- Max frame rate: 60 fps (recommended) +- Supported formats: MP4, MKV, MOV, AVI, FLV, MPEG-2 TS/PS, MXF, LXF, GXF, 3GP, WebM, MPG, QuickTime + +## Pricing + +- $5/1000 min stored +- $1/1000 min delivered + +## Resources + +- Dashboard: https://dash.cloudflare.com/?to=/:account/stream +- API Docs: https://developers.cloudflare.com/api/resources/stream/ +- Stream Docs: https://developers.cloudflare.com/stream/ + +## Reading Order + +| Order | File | Purpose | When to Use | +|-------|------|---------|-------------| +| 1 | [configuration.md](./configuration.md) | Setup SDKs, env vars, signing keys | Starting new project | +| 2 | [api.md](./api.md) | On-demand video APIs | Implementing uploads/playback | +| 3 | [api-live.md](./api-live.md) | Live streaming APIs | Building live streaming | +| 4 | [patterns.md](./patterns.md) | Full-stack flows, TUS, JWT signing | Implementing workflows | +| 5 | [gotchas.md](./gotchas.md) | Errors, limits, troubleshooting | Debugging issues | + +## In This Reference + +- [configuration.md](./configuration.md) - Setup, environment variables, wrangler config +- [api.md](./api.md) - On-demand video upload, playback, management APIs +- [api-live.md](./api-live.md) - Live streaming (RTMPS/SRT/WebRTC), simulcast +- [patterns.md](./patterns.md) - Full-stack flows, state management, best practices +- [gotchas.md](./gotchas.md) - Error codes, troubleshooting, limits + +## See Also + +- [workers](../workers/) - Deploy Stream APIs in Workers +- [pages](../pages/) - Integrate Stream with Pages +- [workers-ai](../workers-ai/) - AI-generate captions diff --git a/plugins/cloudflare/skills/cloudflare/references/stream/api-live.md b/plugins/cloudflare/skills/cloudflare/references/stream/api-live.md new file mode 100644 index 0000000..6c4f4c0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/stream/api-live.md @@ -0,0 +1,195 @@ +# Stream Live Streaming API + +Live input creation, status checking, simulcast, and WebRTC streaming. + +## Create Live Input + +### Using Cloudflare SDK + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: env.CF_API_TOKEN }); + +const liveInput = await client.stream.liveInputs.create({ + account_id: env.CF_ACCOUNT_ID, + recording: { mode: 'automatic', timeoutSeconds: 30 }, + deleteRecordingAfterDays: 30 +}); + +// Returns: { uid, rtmps, srt, webRTC } +``` + +### Raw fetch API + +```typescript +async function createLiveInput(accountId: string, apiToken: string) { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/live_inputs`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + recording: { mode: 'automatic', timeoutSeconds: 30 }, + deleteRecordingAfterDays: 30 + }) + } + ); + const { result } = await response.json(); + return { + uid: result.uid, + rtmps: { url: result.rtmps.url, streamKey: result.rtmps.streamKey }, + srt: { url: result.srt.url, streamId: result.srt.streamId, passphrase: result.srt.passphrase }, + webRTC: result.webRTC + }; +} +``` + +## Check Live Status + +```typescript +async function getLiveStatus(accountId: string, liveInputId: string, apiToken: string) { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/live_inputs/${liveInputId}`, + { headers: { 'Authorization': `Bearer ${apiToken}` } } + ); + const { result } = await response.json(); + return { + isLive: result.status?.current?.state === 'connected', + recording: result.recording, + status: result.status + }; +} +``` + +## Simulcast (Live Outputs) + +### Create Output + +```typescript +async function createLiveOutput( + accountId: string, liveInputId: string, apiToken: string, + outputUrl: string, streamKey: string +) { + return fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/live_inputs/${liveInputId}/outputs`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url: `${outputUrl}/${streamKey}`, + enabled: true, + streamKey // For platforms like YouTube, Twitch + }) + } + ).then(r => r.json()); +} +``` + +### Example: Simulcast to YouTube + Twitch + +```typescript +const liveInput = await createLiveInput(accountId, apiToken); + +// Add YouTube output +await createLiveOutput( + accountId, liveInput.uid, apiToken, + 'rtmp://a.rtmp.youtube.com/live2', + 'your-youtube-stream-key' +); + +// Add Twitch output +await createLiveOutput( + accountId, liveInput.uid, apiToken, + 'rtmp://live.twitch.tv/app', + 'your-twitch-stream-key' +); +``` + +## WebRTC Streaming (WHIP/WHEP) + +### Browser to Stream (WHIP) + +```typescript +async function startWebRTCBroadcast(liveInputId: string) { + const pc = new RTCPeerConnection(); + + // Add local media tracks + const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true }); + stream.getTracks().forEach(track => pc.addTrack(track, stream)); + + // Create offer + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + + // Send to Stream via WHIP + const response = await fetch( + `https://customer-.cloudflarestream.com/${liveInputId}/webRTC/publish`, + { + method: 'POST', + headers: { 'Content-Type': 'application/sdp' }, + body: offer.sdp + } + ); + + const answer = await response.text(); + await pc.setRemoteDescription({ type: 'answer', sdp: answer }); +} +``` + +### Stream to Browser (WHEP) + +```typescript +async function playWebRTCStream(videoId: string) { + const pc = new RTCPeerConnection(); + + pc.addTransceiver('video', { direction: 'recvonly' }); + pc.addTransceiver('audio', { direction: 'recvonly' }); + + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + + const response = await fetch( + `https://customer-.cloudflarestream.com/${videoId}/webRTC/play`, + { + method: 'POST', + headers: { 'Content-Type': 'application/sdp' }, + body: offer.sdp + } + ); + + const answer = await response.text(); + await pc.setRemoteDescription({ type: 'answer', sdp: answer }); + + return pc; +} +``` + +## Recording Settings + +| Mode | Behavior | +|------|----------| +| `automatic` | Record all live streams | +| `off` | No recording | +| `timeoutSeconds` | Stop recording after N seconds of inactivity | + +```typescript +const recordingConfig = { + mode: 'automatic', + timeoutSeconds: 30, // Auto-stop 30s after stream ends + requireSignedURLs: true, // Require token for VOD playback + allowedOrigins: ['https://yourdomain.com'] +}; +``` + +## In This Reference + +- [README.md](./README.md) - Overview and quick start +- [api.md](./api.md) - On-demand video APIs +- [configuration.md](./configuration.md) - Setup and config +- [patterns.md](./patterns.md) - Full-stack flows, best practices +- [gotchas.md](./gotchas.md) - Error codes, troubleshooting + +## See Also + +- [workers](../workers/) - Deploy live APIs in Workers diff --git a/plugins/cloudflare/skills/cloudflare/references/stream/api.md b/plugins/cloudflare/skills/cloudflare/references/stream/api.md new file mode 100644 index 0000000..0c35a71 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/stream/api.md @@ -0,0 +1,199 @@ +# Stream API Reference + +Upload, playback, live streaming, and management APIs. + +## Upload APIs + +### Direct Creator Upload (Recommended) + +**Backend: Create upload URL (SDK)** +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: env.CF_API_TOKEN }); + +const uploadData = await client.stream.directUpload.create({ + account_id: env.CF_ACCOUNT_ID, + maxDurationSeconds: 3600, + requireSignedURLs: true, + meta: { creator: 'user-123' } +}); +// Returns: { uploadURL: string, uid: string } +``` + +**Frontend: Upload file** +```typescript +async function uploadVideo(file: File, uploadURL: string) { + const formData = new FormData(); + formData.append('file', file); + return fetch(uploadURL, { method: 'POST', body: formData }).then(r => r.json()); +} +``` + +### Upload from URL + +```typescript +const video = await client.stream.copy.create({ + account_id: env.CF_ACCOUNT_ID, + url: 'https://example.com/video.mp4', + meta: { name: 'My Video' }, + requireSignedURLs: false +}); +``` + +## Playback APIs + +### Embed Player (iframe) + +```html + +``` + +### HLS/DASH Manifest URLs + +```typescript +// HLS +const hlsUrl = `https://customer-.cloudflarestream.com/${videoId}/manifest/video.m3u8`; + +// DASH +const dashUrl = `https://customer-.cloudflarestream.com/${videoId}/manifest/video.mpd`; +``` + +### Thumbnails + +```typescript +// At specific time (seconds) +const thumb = `https://customer-.cloudflarestream.com/${videoId}/thumbnails/thumbnail.jpg?time=10s`; + +// By percentage +const thumbPct = `https://customer-.cloudflarestream.com/${videoId}/thumbnails/thumbnail.jpg?time=50%`; + +// Animated GIF +const gif = `https://customer-.cloudflarestream.com/${videoId}/thumbnails/thumbnail.gif`; +``` + +## Signed URLs + +```typescript +// Low volume (<1k/day): Use API +async function getSignedToken(accountId: string, videoId: string, apiToken: string) { + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/${videoId}/token`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + exp: Math.floor(Date.now() / 1000) + 3600, + accessRules: [{ type: 'ip.geoip.country', action: 'allow', country: ['US'] }] + }) + } + ); + return (await response.json()).result.token; +} + +// High volume: Self-sign with RS256 JWT (see "Self-Sign JWT" in patterns.md) +``` + +## Captions & Clips + +### Upload Captions + +```typescript +async function uploadCaption( + accountId: string, videoId: string, apiToken: string, + language: string, captionFile: File +) { + const formData = new FormData(); + formData.append('file', captionFile); + return fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/${videoId}/captions/${language}`, + { + method: 'PUT', + headers: { 'Authorization': `Bearer ${apiToken}` }, + body: formData + } + ).then(r => r.json()); +} +``` + +### Generate AI Captions + +```typescript +// TODO: Requires Workers AI integration - see workers-ai reference +async function generateAICaptions(accountId: string, videoId: string, apiToken: string) { + return fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/${videoId}/captions/generate`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ language: 'en' }) + } + ).then(r => r.json()); +} +``` + +### Clip Video + +```typescript +async function clipVideo( + accountId: string, videoId: string, apiToken: string, + startTime: number, endTime: number +) { + return fetch( + `https://api.cloudflare.com/client/v4/accounts/${accountId}/stream/clip`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${apiToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + clippedFromVideoUID: videoId, + startTimeSeconds: startTime, + endTimeSeconds: endTime + }) + } + ).then(r => r.json()); +} +``` + +## Video Management + +```typescript +// List videos +const videos = await client.stream.videos.list({ + account_id: env.CF_ACCOUNT_ID, + search: 'keyword' // optional +}); + +// Get video details +const video = await client.stream.videos.get(videoId, { + account_id: env.CF_ACCOUNT_ID +}); + +// Update video +await client.stream.videos.update(videoId, { + account_id: env.CF_ACCOUNT_ID, + meta: { title: 'New Title' }, + requireSignedURLs: true +}); + +// Delete video +await client.stream.videos.delete(videoId, { + account_id: env.CF_ACCOUNT_ID +}); +``` + +## In This Reference + +- [README.md](./README.md) - Overview and quick start +- [configuration.md](./configuration.md) - Setup and config +- [api-live.md](./api-live.md) - Live streaming APIs (RTMPS/SRT/WebRTC) +- [patterns.md](./patterns.md) - Full-stack flows, best practices +- [gotchas.md](./gotchas.md) - Error codes, troubleshooting + +## See Also + +- [workers](../workers/) - Deploy Stream APIs in Workers diff --git a/plugins/cloudflare/skills/cloudflare/references/stream/configuration.md b/plugins/cloudflare/skills/cloudflare/references/stream/configuration.md new file mode 100644 index 0000000..c4e6613 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/stream/configuration.md @@ -0,0 +1,141 @@ +# Stream Configuration + +Setup, environment variables, and wrangler configuration. + +## Installation + +```bash +# Official Cloudflare SDK (Node.js, Workers, Pages) +npm install cloudflare + +# React component library +npm install @cloudflare/stream-react + +# TUS resumable uploads (large files) +npm install tus-js-client +``` + +## Environment Variables + +```bash +# Required +CF_ACCOUNT_ID=your-account-id +CF_API_TOKEN=your-api-token + +# For signed URLs (high volume) +STREAM_KEY_ID=your-key-id +STREAM_JWK=base64-encoded-jwk + +# For webhooks +WEBHOOK_SECRET=your-webhook-secret + +# Customer subdomain (from dashboard) +STREAM_CUSTOMER_CODE=your-customer-code +``` + +## Wrangler Configuration + +```jsonc +{ + "name": "stream-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + "vars": { + "CF_ACCOUNT_ID": "your-account-id" + } + // Store secrets: wrangler secret put CF_API_TOKEN + // wrangler secret put STREAM_KEY_ID + // wrangler secret put STREAM_JWK + // wrangler secret put WEBHOOK_SECRET +} +``` + +## Signing Keys (High Volume) + +Create once for self-signing tokens (thousands of daily users). + +**Create key** +```bash +curl -X POST \ + "https://api.cloudflare.com/client/v4/accounts/{account_id}/stream/keys" \ + -H "Authorization: Bearer " + +# Save `id` and `jwk` (base64) from response +``` + +**Store in secrets** +```bash +wrangler secret put STREAM_KEY_ID +wrangler secret put STREAM_JWK +``` + +## Webhooks + +**Setup webhook URL** +```bash +curl -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/{account_id}/stream/webhook" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"notificationUrl": "https://your-worker.workers.dev/webhook"}' + +# Save the returned `secret` for signature verification +``` + +**Store secret** +```bash +wrangler secret put WEBHOOK_SECRET +``` + +## Direct Upload / Live / Watermark Config + +```typescript +// Direct upload +const uploadConfig = { + maxDurationSeconds: 3600, + expiry: new Date(Date.now() + 3600000).toISOString(), + requireSignedURLs: true, + allowedOrigins: ['https://yourdomain.com'], + meta: { creator: 'user-123' } +}; + +// Live input +const liveConfig = { + recording: { mode: 'automatic', timeoutSeconds: 30 }, + deleteRecordingAfterDays: 30 +}; + +// Watermark +const watermark = { + name: 'Logo', opacity: 0.7, padding: 20, + position: 'lowerRight', scale: 0.15 +}; +``` + +## Access Rules & Player Config + +```typescript +// Access rules: allow US/CA, block CN/RU, or IP allowlist +const geoRestrict = [ + { type: 'ip.geoip.country', action: 'allow', country: ['US', 'CA'] }, + { type: 'any', action: 'block' } +]; + +// Player params for iframe +const playerParams = new URLSearchParams({ + autoplay: 'true', muted: 'true', preload: 'auto', defaultTextTrack: 'en' +}); +``` + +## In This Reference + +- [README.md](./README.md) - Overview and quick start +- [api.md](./api.md) - On-demand video APIs +- [api-live.md](./api-live.md) - Live streaming APIs +- [patterns.md](./patterns.md) - Full-stack flows, best practices +- [gotchas.md](./gotchas.md) - Error codes, troubleshooting + +## See Also + +- [wrangler](../wrangler/) - Wrangler CLI and configuration +- [workers](../workers/) - Deploy Stream APIs in Workers diff --git a/plugins/cloudflare/skills/cloudflare/references/stream/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/stream/gotchas.md new file mode 100644 index 0000000..2b1cf8b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/stream/gotchas.md @@ -0,0 +1,130 @@ +# Stream Gotchas + +## Common Errors + +### "ERR_NON_VIDEO" + +**Cause:** Uploaded file is not a valid video format +**Solution:** Ensure file is in supported format (MP4, MKV, MOV, AVI, FLV, MPEG-2 TS/PS, MXF, LXF, GXF, 3GP, WebM, MPG, QuickTime) + +### "ERR_DURATION_EXCEED_CONSTRAINT" + +**Cause:** Video duration exceeds `maxDurationSeconds` constraint +**Solution:** Increase `maxDurationSeconds` in direct upload config or trim video before upload + +### "ERR_FETCH_ORIGIN_ERROR" + +**Cause:** Failed to download video from URL (upload from URL) +**Solution:** Ensure URL is publicly accessible, uses HTTPS, and video file is available + +### "ERR_MALFORMED_VIDEO" + +**Cause:** Video file is corrupted or improperly encoded +**Solution:** Re-encode video using FFmpeg or check source file integrity + +### "ERR_DURATION_TOO_SHORT" + +**Cause:** Video must be at least 0.1 seconds long +**Solution:** Ensure video has valid duration (not a single frame) + +## Troubleshooting + +### Video stuck in "inprogress" state +- **Cause**: Processing large/complex video +- **Solution**: Wait up to 5 minutes for processing; use webhooks instead of polling + +### Signed URL returns 403 +- **Cause**: Token expired or invalid signature +- **Solution**: Check expiration timestamp, verify JWK is correct, ensure clock sync + +### Live stream not connecting +- **Cause**: Invalid RTMPS URL or stream key +- **Solution**: Use exact URL/key from API, ensure firewall allows outbound 443 + +### Webhook signature verification fails +- **Cause**: Incorrect secret or timestamp window +- **Solution**: Use exact secret from webhook setup, allow 5-minute timestamp drift + +### Video uploads but isn't visible +- **Cause**: `requireSignedURLs` enabled without providing token +- **Solution**: Generate signed token or set `requireSignedURLs: false` for public videos + +### Player shows infinite loading +- **Cause**: CORS issue with allowedOrigins +- **Solution**: Add your domain to `allowedOrigins` array + +## Limits + +| Resource | Limit | +|----------|-------| +| Max file size | 30 GB | +| Max frame rate | 60 fps (recommended) | +| Max duration per direct upload | Configurable via `maxDurationSeconds` | +| Token generation (API endpoint) | 1,000/day recommended (use signing keys for higher) | +| Live input outputs (simulcast) | 5 per live input | +| Webhook retry attempts | 5 (exponential backoff) | +| Webhook timeout | 30 seconds | +| Caption file size | 5 MB | +| Watermark image size | 2 MB | +| Metadata keys per video | Unlimited | +| Search results per page | Max 1,000 | + +## Performance Issues + +### Upload is slow +- **Cause**: Large file size or network constraints +- **Solution**: Use TUS resumable upload, compress video before upload, check bandwidth + +### Playback buffering +- **Cause**: Network congestion or low bandwidth +- **Solution**: Use ABR (adaptive bitrate) with HLS/DASH, reduce max bitrate + +### High processing time +- **Cause**: Complex video codec, high resolution +- **Solution**: Pre-encode with H.264 (most efficient), reduce resolution + +## Type Safety + +```typescript +// Error response type +interface StreamError { + success: false; + errors: Array<{ + code: number; + message: string; + }>; +} + +// Handle errors +async function uploadWithErrorHandling(url: string, file: File) { + const formData = new FormData(); + formData.append('file', file); + const response = await fetch(url, { method: 'POST', body: formData }); + const result = await response.json(); + + if (!result.success) { + throw new Error(result.errors[0]?.message || 'Upload failed'); + } + return result; +} +``` + +## Security Gotchas + +1. **Never expose API token in frontend** - Use direct creator uploads +2. **Always verify webhook signatures** - Prevent spoofed notifications +3. **Set appropriate token expiration** - Short-lived for security +4. **Use requireSignedURLs for private content** - Prevent unauthorized access +5. **Whitelist allowedOrigins** - Prevent hotlinking/embedding on unauthorized sites + +## In This Reference + +- [README.md](./README.md) - Overview and quick start +- [configuration.md](./configuration.md) - Setup and config +- [api.md](./api.md) - On-demand video APIs +- [api-live.md](./api-live.md) - Live streaming APIs +- [patterns.md](./patterns.md) - Full-stack flows, best practices + +## See Also + +- [workers](../workers/) - Deploy Stream APIs securely diff --git a/plugins/cloudflare/skills/cloudflare/references/stream/patterns.md b/plugins/cloudflare/skills/cloudflare/references/stream/patterns.md new file mode 100644 index 0000000..2e7782d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/stream/patterns.md @@ -0,0 +1,184 @@ +# Stream Patterns + +Common workflows, full-stack flows, and best practices. + +## React Stream Player + +`npm install @cloudflare/stream-react` + +```tsx +import { Stream } from '@cloudflare/stream-react'; + +export function VideoPlayer({ videoId, token }: { videoId: string; token?: string }) { + return ; +} +``` + +## Full-Stack Upload Flow + +**Backend API (Workers/Pages)** +```typescript +import Cloudflare from 'cloudflare'; + +export default { + async fetch(request: Request, env: Env): Promise { + const { videoName } = await request.json(); + const client = new Cloudflare({ apiToken: env.CF_API_TOKEN }); + const { uploadURL, uid } = await client.stream.directUpload.create({ + account_id: env.CF_ACCOUNT_ID, + maxDurationSeconds: 3600, + requireSignedURLs: true, + meta: { name: videoName } + }); + return Response.json({ uploadURL, uid }); + } +}; +``` + +**Frontend component** +```tsx +import { useState } from 'react'; + +export function VideoUploader() { + const [uploading, setUploading] = useState(false); + const [progress, setProgress] = useState(0); + + async function handleUpload(file: File) { + setUploading(true); + const { uploadURL, uid } = await fetch('/api/upload-url', { + method: 'POST', + body: JSON.stringify({ videoName: file.name }) + }).then(r => r.json()); + + const xhr = new XMLHttpRequest(); + xhr.upload.onprogress = (e) => setProgress((e.loaded / e.total) * 100); + xhr.onload = () => { setUploading(false); window.location.href = `/videos/${uid}`; }; + xhr.open('POST', uploadURL); + const formData = new FormData(); + formData.append('file', file); + xhr.send(formData); + } + + return ( +
+ e.target.files?.[0] && handleUpload(e.target.files[0])} disabled={uploading} /> + {uploading && } +
+ ); +} +``` + +## TUS Resumable Upload + +For large files (>500MB). `npm install tus-js-client` + +```typescript +import * as tus from 'tus-js-client'; + +async function uploadWithTUS(file: File, uploadURL: string, onProgress?: (pct: number) => void) { + return new Promise((resolve, reject) => { + const upload = new tus.Upload(file, { + endpoint: uploadURL, + retryDelays: [0, 3000, 5000, 10000, 20000], + chunkSize: 50 * 1024 * 1024, + metadata: { filename: file.name, filetype: file.type }, + onError: reject, + onProgress: (up, total) => onProgress?.((up / total) * 100), + onSuccess: () => resolve(upload.url?.split('/').pop() || '') + }); + upload.start(); + }); +} +``` + +## Video State Polling + +```typescript +async function waitForVideoReady(client: Cloudflare, accountId: string, videoId: string) { + for (let i = 0; i < 60; i++) { + const video = await client.stream.videos.get(videoId, { account_id: accountId }); + if (video.readyToStream || video.status.state === 'error') return video; + await new Promise(resolve => setTimeout(resolve, 5000)); + } + throw new Error('Video processing timeout'); +} +``` + +## Webhook Handler + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const signature = request.headers.get('Webhook-Signature'); + const body = await request.text(); + if (!signature || !await verifyWebhook(signature, body, env.WEBHOOK_SECRET)) { + return new Response('Unauthorized', { status: 401 }); + } + const payload = JSON.parse(body); + if (payload.readyToStream) console.log(`Video ${payload.uid} ready`); + return new Response('OK'); + } +}; + +async function verifyWebhook(sig: string, body: string, secret: string): Promise { + const parts = Object.fromEntries(sig.split(',').map(p => p.split('='))); + const timestamp = parseInt(parts.time || '0', 10); + if (Math.abs(Date.now() / 1000 - timestamp) > 300) return false; + + const key = await crypto.subtle.importKey( + 'raw', new TextEncoder().encode(secret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'] + ); + const computed = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(`${timestamp}.${body}`)); + const hex = Array.from(new Uint8Array(computed), b => b.toString(16).padStart(2, '0')).join(''); + return hex === parts.sig1; +} +``` + +## Self-Sign JWT (High Volume Tokens) + +For >1k tokens/day. Prerequisites: Create signing key (see configuration.md). + +```typescript +async function selfSignToken(keyId: string, jwkBase64: string, videoId: string, expiresIn = 3600) { + const key = await crypto.subtle.importKey( + 'jwk', JSON.parse(atob(jwkBase64)), { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' }, false, ['sign'] + ); + const now = Math.floor(Date.now() / 1000); + const header = btoa(JSON.stringify({ alg: 'RS256', kid: keyId })).replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_'); + const payload = btoa(JSON.stringify({ sub: videoId, kid: keyId, exp: now + expiresIn, nbf: now })) + .replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_'); + const message = `${header}.${payload}`; + const sig = await crypto.subtle.sign('RSASSA-PKCS1-v1_5', key, new TextEncoder().encode(message)); + const b64Sig = btoa(String.fromCharCode(...new Uint8Array(sig))).replace(/=/g, '').replace(/\+/g, '-').replace(/\//g, '_'); + return `${message}.${b64Sig}`; +} + +// With access rules (geo-restriction) +const payloadWithRules = { + sub: videoId, kid: keyId, exp: now + 3600, nbf: now, + accessRules: [{ type: 'ip.geoip.country', action: 'allow', country: ['US'] }] +}; +``` + +## Best Practices + +- **Use Direct Creator Uploads** - Avoid proxying through servers +- **Enable requireSignedURLs** - Control private content access +- **Self-sign tokens at scale** - Use signing keys for >1k/day +- **Set allowedOrigins** - Prevent hotlinking +- **Use webhooks over polling** - Efficient status updates +- **Set maxDurationSeconds** - Prevent abuse +- **Enable live recordings** - Auto VOD after stream + +## In This Reference + +- [README.md](./README.md) - Overview and quick start +- [configuration.md](./configuration.md) - Setup and config +- [api.md](./api.md) - On-demand video APIs +- [api-live.md](./api-live.md) - Live streaming APIs +- [gotchas.md](./gotchas.md) - Error codes, troubleshooting + +## See Also + +- [workers](../workers/) - Deploy Stream APIs in Workers +- [pages](../pages/) - Integrate Stream with Pages diff --git a/plugins/cloudflare/skills/cloudflare/references/tail-workers/README.md b/plugins/cloudflare/skills/cloudflare/references/tail-workers/README.md new file mode 100644 index 0000000..d17da7d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tail-workers/README.md @@ -0,0 +1,89 @@ +# Cloudflare Tail Workers + +Specialized Workers that consume execution events from producer Workers for logging, debugging, analytics, and observability. + +## When to Use This Reference + +- Implementing observability/logging for Cloudflare Workers +- Processing Worker execution events, logs, exceptions +- Building custom analytics or error tracking +- Configuring real-time event streaming +- Working with tail handlers or tail consumers + +## Core Concepts + +### What Are Tail Workers? + +Tail Workers automatically process events from producer Workers (the Workers being monitored). They receive: +- HTTP request/response info +- Console logs (`console.log/error/warn/debug`) +- Uncaught exceptions +- Execution outcomes (`ok`, `exception`, `exceededCpu`, etc.) +- Diagnostic channel events + +**Key characteristics:** +- Invoked AFTER producer finishes executing +- Capture entire request lifecycle including Service Bindings and Dynamic Dispatch sub-requests +- Billed by CPU time, not request count +- Available on Workers Paid and Enterprise tiers + +### Alternative: OpenTelemetry Export + +**Before using Tail Workers, consider OpenTelemetry:** + +For batch exports to observability tools (Sentry, Grafana, Honeycomb): +- OTEL export sends logs/traces in batches (more efficient) +- Built-in integrations with popular platforms +- Lower overhead than Tail Workers +- **Use Tail Workers only for custom real-time processing** + +## Decision Tree + +``` +Need observability for Workers? +├─ Batch export to known tools (Sentry/Grafana/Honeycomb)? +│ └─ Use OpenTelemetry export (not Tail Workers) +├─ Custom real-time processing needed? +│ ├─ Aggregated metrics? +│ │ └─ Use Tail Worker + Analytics Engine +│ ├─ Error tracking? +│ │ └─ Use Tail Worker + external service +│ ├─ Custom logging/debugging? +│ │ └─ Use Tail Worker + KV/HTTP endpoint +│ └─ Complex event processing? +│ └─ Use Tail Worker + Durable Objects +└─ Quick debugging? + └─ Use `wrangler tail` (different from Tail Workers) +``` + +## Reading Order + +1. **[configuration.md](configuration.md)** - Set up Tail Workers +2. **[api.md](api.md)** - Handler signature, types, redaction +3. **[patterns.md](patterns.md)** - Common use cases and integrations +4. **[gotchas.md](gotchas.md)** - Pitfalls and debugging tips + +## Quick Example + +```typescript +export default { + async tail(events, env, ctx) { + // Process events from producer Worker + ctx.waitUntil( + fetch(env.LOG_ENDPOINT, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(events), + }) + ); + } +}; +``` + +## Related Skills + +- **observability** - General Workers observability patterns, OTEL export +- **analytics-engine** - Aggregated metrics storage for tail event data +- **durable-objects** - Stateful event processing, batching tail events +- **logpush** - Alternative for batch log export (non-real-time) +- **workers-for-platforms** - Dynamic dispatch with tail consumers diff --git a/plugins/cloudflare/skills/cloudflare/references/tail-workers/api.md b/plugins/cloudflare/skills/cloudflare/references/tail-workers/api.md new file mode 100644 index 0000000..624d9e9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tail-workers/api.md @@ -0,0 +1,200 @@ +# Tail Workers API Reference + +## Handler Signature + +```typescript +export default { + async tail( + events: TraceItem[], + env: Env, + ctx: ExecutionContext + ): Promise { + // Process events + } +} satisfies ExportedHandler; +``` + +**Parameters:** +- `events`: Array of `TraceItem` objects (one per producer invocation) +- `env`: Bindings (KV, D1, R2, env vars, etc.) +- `ctx`: Context with `waitUntil()` for async work + +**CRITICAL:** Tail handlers don't return values. Use `ctx.waitUntil()` for async operations. + +## TraceItem Type + +```typescript +interface TraceItem { + scriptName: string; // Producer Worker name + eventTimestamp: number; // Epoch milliseconds + outcome: 'ok' | 'exception' | 'exceededCpu' | 'exceededMemory' + | 'canceled' | 'scriptNotFound' | 'responseStreamDisconnected' | 'unknown'; + + event?: { + request?: { + url: string; // Redacted by default + method: string; + headers: Record; // Sensitive headers redacted + cf?: IncomingRequestCfProperties; + getUnredacted(): TraceRequest; // Bypass redaction (use carefully) + }; + response?: { + status: number; + }; + }; + + logs: Array<{ + timestamp: number; // Epoch milliseconds + level: 'debug' | 'info' | 'log' | 'warn' | 'error'; + message: unknown[]; // Args passed to console function + }>; + + exceptions: Array<{ + timestamp: number; // Epoch milliseconds + name: string; // Error type (Error, TypeError, etc.) + message: string; // Error description + }>; + + diagnosticsChannelEvents: Array<{ + channel: string; + message: unknown; + timestamp: number; // Epoch milliseconds + }>; +} +``` + +**Note:** Official SDK uses `TraceItem`, not `TailItem`. Use `@cloudflare/workers-types` for accurate types. + +## Timestamp Handling + +All timestamps are **epoch milliseconds**, not seconds: + +```typescript +// ✅ CORRECT - use directly with Date +const date = new Date(event.eventTimestamp); + +// ❌ WRONG - don't multiply by 1000 +const date = new Date(event.eventTimestamp * 1000); +``` + +## Automatic Redaction + +By default, sensitive data is redacted from `TraceRequest`: + +### Header Redaction + +Headers containing these substrings (case-insensitive): +- `auth`, `key`, `secret`, `token`, `jwt` +- `cookie`, `set-cookie` + +Redacted values show as `"REDACTED"`. + +### URL Redaction + +- **Hex IDs:** 32+ hex digits → `"REDACTED"` +- **Base-64 IDs:** 21+ chars with 2+ upper, 2+ lower, 2+ digits → `"REDACTED"` + +## Bypassing Redaction + +```typescript +export default { + async tail(events, env, ctx) { + for (const event of events) { + // ⚠️ Use with extreme caution + const unredacted = event.event?.request?.getUnredacted(); + // unredacted.url and unredacted.headers contain raw values + } + } +}; +``` + +**Best practices:** +- Only call `getUnredacted()` when absolutely necessary +- Never log unredacted sensitive data +- Implement additional filtering before external transmission +- Use environment variables for API keys, never hardcode + +## Type-Safe Handler + +```typescript +interface Env { + LOGS_KV: KVNamespace; + ANALYTICS: AnalyticsEngineDataset; + LOG_ENDPOINT: string; + API_TOKEN: string; +} + +export default { + async tail( + events: TraceItem[], + env: Env, + ctx: ExecutionContext + ): Promise { + const payload = events.map(event => ({ + script: event.scriptName, + timestamp: event.eventTimestamp, + outcome: event.outcome, + url: event.event?.request?.url, + status: event.event?.response?.status, + })); + + ctx.waitUntil( + fetch(env.LOG_ENDPOINT, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }) + ); + } +} satisfies ExportedHandler; +``` + +## Outcome vs HTTP Status + +**IMPORTANT:** `outcome` is script execution status, NOT HTTP status. + +- Worker returns 500 → `outcome='ok'` if script completed successfully +- Uncaught exception → `outcome='exception'` regardless of HTTP status +- CPU limit exceeded → `outcome='exceededCpu'` + +```typescript +// ✅ Check outcome for script execution status +if (event.outcome === 'exception') { + // Script threw uncaught exception +} + +// ✅ Check HTTP status separately +if (event.event?.response?.status === 500) { + // HTTP 500 returned (script may have handled error) +} +``` + +## Serialization Considerations + +`log.message` is `unknown[]` and may contain non-serializable objects: + +```typescript +// ❌ May fail with circular references or BigInt +JSON.stringify(events); + +// ✅ Safe serialization +const safePayload = events.map(event => ({ + ...event, + logs: event.logs.map(log => ({ + ...log, + message: log.message.map(m => { + try { + return JSON.parse(JSON.stringify(m)); + } catch { + return String(m); + } + }) + })) +})); +``` + +**Common serialization issues:** +- Circular references in logged objects +- `BigInt` values (not JSON-serializable) +- Functions or symbols in console.log arguments +- Large objects exceeding body size limits diff --git a/plugins/cloudflare/skills/cloudflare/references/tail-workers/configuration.md b/plugins/cloudflare/skills/cloudflare/references/tail-workers/configuration.md new file mode 100644 index 0000000..96fb33f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tail-workers/configuration.md @@ -0,0 +1,176 @@ +# Tail Workers Configuration + +## Setup Steps + +### 1. Create Tail Worker + +Create a Worker with a `tail()` handler: + +```typescript +export default { + async tail(events, env, ctx) { + // Process events from producer Worker + ctx.waitUntil( + fetch(env.LOG_ENDPOINT, { + method: "POST", + body: JSON.stringify(events), + }) + ); + } +}; +``` + +### 2. Configure Producer Worker + +In producer's `wrangler.jsonc`: + +```jsonc +{ + "name": "my-producer-worker", + "tail_consumers": [ + { + "service": "my-tail-worker" + } + ] +} +``` + +### 3. Deploy Both Workers + +```bash +# Deploy Tail Worker first +cd tail-worker +wrangler deploy + +# Then deploy producer Worker +cd ../producer-worker +wrangler deploy +``` + +## Wrangler Configuration + +### Single Tail Consumer + +```jsonc +{ + "name": "producer-worker", + "tail_consumers": [ + { + "service": "logging-tail-worker" + } + ] +} +``` + +### Multiple Tail Consumers + +```jsonc +{ + "name": "producer-worker", + "tail_consumers": [ + { + "service": "logging-tail-worker" + }, + { + "service": "metrics-tail-worker" + } + ] +} +``` + +**Note:** Each consumer receives ALL events independently. + +### Remove Tail Consumer + +```jsonc +{ + "tail_consumers": [] +} +``` + +Then redeploy producer Worker. + +## Environment Variables + +Tail Workers use same binding syntax as regular Workers: + +```jsonc +{ + "name": "my-tail-worker", + "vars": { + "LOG_ENDPOINT": "https://logs.example.com/ingest" + }, + "kv_namespaces": [ + { + "binding": "LOGS_KV", + "id": "abc123..." + } + ] +} +``` + +## Testing & Development + +### Local Testing + +**Tail Workers cannot be fully tested with `wrangler dev`.** Deploy to staging environment for testing. + +### Testing Strategy + +1. Deploy producer Worker to staging +2. Deploy Tail Worker to staging +3. Configure `tail_consumers` in producer +4. Trigger producer Worker requests +5. Verify Tail Worker receives events (check destination logs/storage) + +### Wrangler Tail Command + +```bash +# Stream logs to terminal (NOT Tail Workers) +wrangler tail my-producer-worker +``` + +**This is different from Tail Workers:** +- `wrangler tail` streams logs to your terminal +- Tail Workers are Workers that process events programmatically + +## Deployment Checklist + +- [ ] Tail Worker has `tail()` handler +- [ ] Tail Worker deployed before producer +- [ ] Producer's `wrangler.jsonc` has correct `tail_consumers` +- [ ] Environment variables configured +- [ ] Tested with staging environment +- [ ] Monitoring configured for Tail Worker itself + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Max tail consumers per producer | 10 | Each receives all events independently | +| Events batch size | Up to 100 events per invocation | Larger batches split across invocations | +| Tail Worker CPU time | Same as regular Workers | 10ms (free), 30ms (paid), 50ms (paid bundle) | +| Pricing tier | Workers Paid or Enterprise | Not available on free plan | +| Request body size | 100 MB max | When sending to external endpoints | +| Event retention | None | Events not retried if tail handler fails | + +## Workers for Platforms + +For dynamic dispatch Workers, both dispatch and user Worker events sent to tail consumer: + +```jsonc +{ + "name": "dispatch-worker", + "tail_consumers": [ + { + "service": "platform-tail-worker" + } + ] +} +``` + +Tail Worker receives TWO `TraceItem` elements per request: +1. Dynamic dispatch Worker event +2. User Worker event + +See [patterns.md](patterns.md) for handling. diff --git a/plugins/cloudflare/skills/cloudflare/references/tail-workers/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/tail-workers/gotchas.md new file mode 100644 index 0000000..4865d0e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tail-workers/gotchas.md @@ -0,0 +1,192 @@ +# Tail Workers Gotchas & Debugging + +## Critical Pitfalls + +### 1. Not Using `ctx.waitUntil()` + +**Problem:** Async work doesn't complete or tail Worker times out +**Cause:** Handlers exit immediately; awaiting blocks processing +**Solution:** + +```typescript +// ❌ WRONG - fire and forget +export default { + async tail(events) { + fetch(endpoint, { body: JSON.stringify(events) }); + } +}; + +// ❌ WRONG - blocking await +export default { + async tail(events, env, ctx) { + await fetch(endpoint, { body: JSON.stringify(events) }); + } +}; + +// ✅ CORRECT +export default { + async tail(events, env, ctx) { + ctx.waitUntil( + (async () => { + await fetch(endpoint, { body: JSON.stringify(events) }); + await processMore(); + })() + ); + } +}; +``` + +### 2. Missing `tail()` Handler + +**Problem:** Producer deployment fails +**Cause:** Worker in `tail_consumers` doesn't export `tail()` handler +**Solution:** Ensure `export default { async tail(events, env, ctx) { ... } }` + +### 3. Outcome vs HTTP Status + +**Problem:** Filtering by wrong status +**Cause:** `outcome` is script execution status, not HTTP status + +```typescript +// ❌ WRONG +if (event.outcome === 500) { /* never matches */ } + +// ✅ CORRECT +if (event.outcome === 'exception') { /* script threw */ } +if (event.event?.response?.status === 500) { /* HTTP 500 */ } +``` + +### 4. Timestamp Units + +**Problem:** Dates off by 1000x +**Cause:** Timestamps are epoch milliseconds, not seconds + +```typescript +// ❌ WRONG: const date = new Date(event.eventTimestamp * 1000); +// ✅ CORRECT: const date = new Date(event.eventTimestamp); +``` + +### 5. Type Name Mismatch + +**Problem:** Using `TailItem` type +**Cause:** Old docs used `TailItem`, SDK uses `TraceItem` + +```typescript +import type { TraceItem } from '@cloudflare/workers-types'; +export default { + async tail(events: TraceItem[], env, ctx) { /* ... */ } +}; +``` + +### 6. Excessive Logging Volume + +**Problem:** Unexpected high costs +**Cause:** Invoked on EVERY producer request +**Solution:** Sample events + +```typescript +export default { + async tail(events, env, ctx) { + if (Math.random() > 0.1) return; // 10% sample + ctx.waitUntil(sendToEndpoint(events)); + } +}; +``` + +### 7. Serialization Issues + +**Problem:** `JSON.stringify()` fails +**Cause:** `log.message` is `unknown[]` with non-serializable values +**Solution:** + +```typescript +const safePayload = events.map(e => ({ + ...e, + logs: e.logs.map(log => ({ + ...log, + message: log.message.map(m => { + try { return JSON.parse(JSON.stringify(m)); } + catch { return String(m); } + }) + })) +})); +``` + +### 8. Missing Error Handling + +**Problem:** Tail Worker silently fails +**Cause:** No try/catch +**Solution:** + +```typescript +ctx.waitUntil((async () => { + try { + await fetch(env.ENDPOINT, { body: JSON.stringify(events) }); + } catch (error) { + console.error("Tail error:", error); + await env.FALLBACK_KV.put(`failed:${Date.now()}`, JSON.stringify(events)); + } +})()); +``` + +### 9. Deployment Order + +**Problem:** Producer deployment fails +**Cause:** Tail consumer not deployed yet +**Solution:** Deploy tail consumer FIRST + +```bash +cd tail-worker && wrangler deploy +cd ../producer && wrangler deploy +``` + +### 10. No Event Retry + +**Problem:** Events lost when handler fails +**Cause:** Failed invocations NOT retried +**Solution:** Implement fallback storage (see #8) + +## Debugging + +**View logs:** `wrangler tail my-tail-worker` + +**Incremental testing:** +1. Verify receipt: `console.log('Events:', events.length)` +2. Inspect structure: `console.log(JSON.stringify(events[0], null, 2))` +3. Add external call with `ctx.waitUntil()` + +**Monitor dashboard:** Check invocation count (matches producer?), error rate, CPU time + +## Testing + +Add test endpoint to producer: + +```typescript +export default { + async fetch(request) { + if (request.url.includes('/test')) { + console.log('Test log'); + throw new Error('Test error'); + } + return new Response('OK'); + } +}; +``` + +Trigger: `curl https://producer.example.workers.dev/test` + +## Common Errors + +| Error | Cause | Solution | +|-------|-------|----------| +| "Tail consumer not found" | Not deployed | Deploy tail Worker first | +| "No tail handler" | Missing `tail()` | Add to default export | +| "waitUntil is not a function" | Missing `ctx` | Add `ctx` parameter | +| Timeout | Blocking await | Use `ctx.waitUntil()` | + +## Performance Notes + +- Max 100 events per invocation +- Each consumer receives all events independently +- CPU limits same as regular Workers +- For high volume, use Durable Objects batching diff --git a/plugins/cloudflare/skills/cloudflare/references/tail-workers/patterns.md b/plugins/cloudflare/skills/cloudflare/references/tail-workers/patterns.md new file mode 100644 index 0000000..a696ec2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tail-workers/patterns.md @@ -0,0 +1,180 @@ +# Tail Workers Common Patterns + +## Community Libraries + +While most tail Worker implementations are custom, these libraries may help: + +**Logging/Observability:** +- **Axiom** - `axiom-cloudflare-workers` (npm) - Direct Axiom integration +- **Baselime** - SDK for Baselime observability platform +- **LogFlare** - Structured log aggregation + +**Type Definitions:** +- **@cloudflare/workers-types** - Official TypeScript types (use `TraceItem`) + +**Note:** Most integrations require custom tail handler implementation. See integration examples below. + +## Basic Patterns + +### HTTP Endpoint Logging + +```typescript +export default { + async tail(events, env, ctx) { + const payload = events.map(event => ({ + script: event.scriptName, + timestamp: event.eventTimestamp, + outcome: event.outcome, + url: event.event?.request?.url, + status: event.event?.response?.status, + logs: event.logs, + exceptions: event.exceptions, + })); + + ctx.waitUntil( + fetch(env.LOG_ENDPOINT, { + method: "POST", + body: JSON.stringify(payload), + }) + ); + } +}; +``` + +### Error Tracking Only + +```typescript +export default { + async tail(events, env, ctx) { + const errors = events.filter(e => + e.outcome === 'exception' || e.exceptions.length > 0 + ); + + if (errors.length === 0) return; + + ctx.waitUntil( + fetch(env.ERROR_ENDPOINT, { + method: "POST", + body: JSON.stringify(errors), + }) + ); + } +}; +``` + +## Storage Integration + +### KV Storage with TTL + +```typescript +export default { + async tail(events, env, ctx) { + ctx.waitUntil( + Promise.all(events.map(event => + env.LOGS_KV.put( + `log:${event.scriptName}:${event.eventTimestamp}`, + JSON.stringify(event), + { expirationTtl: 86400 } // 24 hours + ) + )) + ); + } +}; +``` + +### Analytics Engine Metrics + +```typescript +export default { + async tail(events, env, ctx) { + ctx.waitUntil( + Promise.all(events.map(event => + env.ANALYTICS.writeDataPoint({ + blobs: [event.scriptName, event.outcome], + doubles: [1, event.event?.response?.status ?? 0], + indexes: [event.event?.request?.cf?.colo ?? 'unknown'], + }) + )) + ); + } +}; +``` + +## Filtering & Routing + +Filter by route, outcome, or other criteria: + +```typescript +export default { + async tail(events, env, ctx) { + // Route filtering + const apiEvents = events.filter(e => + e.event?.request?.url?.includes('/api/') + ); + + // Multi-destination routing + const errors = events.filter(e => e.outcome === 'exception'); + const success = events.filter(e => e.outcome === 'ok'); + + const tasks = []; + if (errors.length > 0) { + tasks.push(fetch(env.ERROR_ENDPOINT, { + method: "POST", + body: JSON.stringify(errors), + })); + } + if (success.length > 0) { + tasks.push(fetch(env.SUCCESS_ENDPOINT, { + method: "POST", + body: JSON.stringify(success), + })); + } + + ctx.waitUntil(Promise.all(tasks)); + } +}; +``` + +## Sampling + +Reduce costs by processing only a percentage of events: + +```typescript +export default { + async tail(events, env, ctx) { + if (Math.random() > 0.1) return; // 10% sample rate + ctx.waitUntil(fetch(env.LOG_ENDPOINT, { + method: "POST", + body: JSON.stringify(events), + })); + } +}; +``` + +## Advanced Patterns + +### Batching with Durable Objects + +Accumulate events before sending: + +```typescript +export default { + async tail(events, env, ctx) { + const batch = env.BATCH_DO.get(env.BATCH_DO.idFromName("batch")); + ctx.waitUntil(batch.fetch("https://batch/add", { + method: "POST", + body: JSON.stringify(events), + })); + } +}; +``` + +See durable-objects skill for full implementation. + +### Workers for Platforms + +Dynamic dispatch sends TWO events per request. Filter by `scriptName` to distinguish dispatch vs user Worker events. + +### Error Handling + +Always wrap external calls. See gotchas.md for fallback storage pattern. diff --git a/plugins/cloudflare/skills/cloudflare/references/terraform/README.md b/plugins/cloudflare/skills/cloudflare/references/terraform/README.md new file mode 100644 index 0000000..17d8a30 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/terraform/README.md @@ -0,0 +1,102 @@ +# Cloudflare Terraform Provider + +**Expert guidance for Cloudflare Terraform Provider - infrastructure as code for Cloudflare resources.** + +## Core Principles + +- **Provider-first**: Use Terraform provider for ALL infrastructure - never mix with wrangler.jsonc for the same resources +- **State management**: Always use remote state (S3, Terraform Cloud, etc.) for team environments +- **Modular architecture**: Create reusable modules for common patterns (zones, workers, pages) +- **Version pinning**: Always pin provider version with `~>` for predictable upgrades +- **Secret management**: Use variables + environment vars for sensitive data - never hardcode API tokens + +## Provider Version + +| Version | Status | Notes | +|---------|--------|-------| +| 5.x | Current | Auto-generated from OpenAPI, breaking changes from v4 | +| 4.x | Legacy | Manual maintenance, deprecated | + +**Critical:** v5 renamed many resources (`cloudflare_record` → `cloudflare_dns_record`, `cloudflare_worker_*` → `cloudflare_workers_*`). See [gotchas.md](./gotchas.md#v5-breaking-changes) for migration details. + +## Provider Setup + +### Basic Configuration + +```hcl +terraform { + required_version = ">= 1.0" + + required_providers { + cloudflare = { + source = "cloudflare/cloudflare" + version = "~> 5.15.0" + } + } +} + +provider "cloudflare" { + api_token = var.cloudflare_api_token # or CLOUDFLARE_API_TOKEN env var +} +``` + +### Authentication Methods (priority order) + +1. **API Token** (RECOMMENDED): `api_token` or `CLOUDFLARE_API_TOKEN` + - Create: Dashboard → My Profile → API Tokens + - Scope to specific accounts/zones for security + +2. **Global API Key** (LEGACY): `api_key` + `api_email` or `CLOUDFLARE_API_KEY` + `CLOUDFLARE_EMAIL` + - Less secure, use tokens instead + +3. **User Service Key**: `user_service_key` for Origin CA certificates + + + +## Quick Reference: Common Commands + +```bash +terraform init # Initialize provider +terraform plan # Plan changes +terraform apply # Apply changes +terraform destroy # Destroy resources +terraform import cloudflare_zone.example # Import existing +terraform state list # List resources in state +terraform output # Show outputs +terraform fmt -recursive # Format code +terraform validate # Validate configuration +``` + +## Import Existing Resources + +Use cf-terraforming to generate configs from existing Cloudflare resources: + +```bash +# Install +brew install cloudflare/cloudflare/cf-terraforming + +# Generate HCL from existing resources +cf-terraforming generate --resource-type cloudflare_dns_record --zone + +# Import into Terraform state +cf-terraforming import --resource-type cloudflare_dns_record --zone +``` + +## Reading Order + +1. Start with [README.md](./README.md) for provider setup and authentication +2. Review [configuration.md](./configuration.md) for resource configurations +3. Check [api.md](./api.md) for data sources and existing resource queries +4. See [patterns.md](./patterns.md) for multi-environment and CI/CD patterns +5. Read [gotchas.md](./gotchas.md) for state drift, v5 breaking changes, and troubleshooting + +## In This Reference +- [configuration.md](./configuration.md) - Resources for zones, DNS, workers, KV, R2, D1, Pages, rulesets +- [api.md](./api.md) - Data sources for existing resources +- [patterns.md](./patterns.md) - Architecture patterns, multi-env setup, CI/CD integration +- [gotchas.md](./gotchas.md) - Common issues, security, best practices + +## See Also +- [pulumi](../pulumi/) - Alternative IaC tool for Cloudflare +- [wrangler](../wrangler/) - CLI deployment alternative +- [workers](../workers/) - Worker runtime documentation diff --git a/plugins/cloudflare/skills/cloudflare/references/terraform/api.md b/plugins/cloudflare/skills/cloudflare/references/terraform/api.md new file mode 100644 index 0000000..8a06c1c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/terraform/api.md @@ -0,0 +1,178 @@ +# Terraform Data Sources Reference + +Query existing Cloudflare resources to reference in your configurations. + +## v5 Data Source Names + +| v4 Name | v5 Name | Notes | +|---------|---------|-------| +| `cloudflare_record` | `cloudflare_dns_record` | | +| `cloudflare_worker_script` | `cloudflare_workers_script` | Note: plural | +| `cloudflare_access_*` | `cloudflare_zero_trust_*` | Access → Zero Trust | + +## Zone Data Sources + +```hcl +# Get zone by name +data "cloudflare_zone" "example" { + name = "example.com" +} + +# Use in resources +resource "cloudflare_dns_record" "www" { + zone_id = data.cloudflare_zone.example.id + name = "www" + # ... +} +``` + +## Account Data Sources + +```hcl +# List all accounts +data "cloudflare_accounts" "main" { + name = "My Account" +} + +# Use account ID +resource "cloudflare_worker_script" "api" { + account_id = data.cloudflare_accounts.main.accounts[0].id + # ... +} +``` + +## Worker Data Sources + +```hcl +# Get existing worker script (v5: cloudflare_workers_script) +data "cloudflare_workers_script" "existing" { + account_id = var.account_id + name = "existing-worker" +} + +# Reference in service bindings +resource "cloudflare_workers_script" "consumer" { + service_binding { + name = "UPSTREAM" + service = data.cloudflare_workers_script.existing.name + } +} +``` + +## KV Data Sources + +```hcl +# Get KV namespace +data "cloudflare_workers_kv_namespace" "existing" { + account_id = var.account_id + namespace_id = "abc123" +} + +# Use in worker binding +resource "cloudflare_workers_script" "api" { + kv_namespace_binding { + name = "KV" + namespace_id = data.cloudflare_workers_kv_namespace.existing.id + } +} +``` + +## Lists Data Source + +```hcl +# Get IP lists for WAF rules +data "cloudflare_list" "blocked_ips" { + account_id = var.account_id + name = "blocked_ips" +} +``` + +## IP Ranges Data Source + +```hcl +# Get Cloudflare IP ranges (for firewall rules) +data "cloudflare_ip_ranges" "cloudflare" {} + +output "ipv4_cidrs" { + value = data.cloudflare_ip_ranges.cloudflare.ipv4_cidr_blocks +} + +output "ipv6_cidrs" { + value = data.cloudflare_ip_ranges.cloudflare.ipv6_cidr_blocks +} + +# Use in security group rules (AWS example) +resource "aws_security_group_rule" "allow_cloudflare" { + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = data.cloudflare_ip_ranges.cloudflare.ipv4_cidr_blocks + security_group_id = aws_security_group.web.id +} +``` + +## Common Patterns + +### Import ID Formats + +| Resource | Import ID Format | +|----------|------------------| +| `cloudflare_zone` | `` | +| `cloudflare_dns_record` | `/` | +| `cloudflare_workers_script` | `/` | +| `cloudflare_workers_kv_namespace` | `/` | +| `cloudflare_r2_bucket` | `/` | +| `cloudflare_d1_database` | `/` | +| `cloudflare_pages_project` | `/` | + +```bash +# Example: Import DNS record +terraform import cloudflare_dns_record.example / +``` + +### Reference Across Modules + +```hcl +# modules/worker/main.tf +data "cloudflare_zone" "main" { + name = var.domain +} + +resource "cloudflare_worker_route" "api" { + zone_id = data.cloudflare_zone.main.id + pattern = "api.${var.domain}/*" + script_name = cloudflare_worker_script.api.name +} +``` + +### Output Important Values + +```hcl +output "zone_id" { + value = cloudflare_zone.main.id + description = "Zone ID for DNS management" +} + +output "worker_url" { + value = "https://${cloudflare_worker_domain.api.hostname}" + description = "Worker API endpoint" +} + +output "kv_namespace_id" { + value = cloudflare_workers_kv_namespace.app.id + sensitive = false +} + +output "name_servers" { + value = cloudflare_zone.main.name_servers + description = "Name servers for domain registration" +} +``` + +## See Also + +- [README](./README.md) - Provider setup +- [Configuration Reference](./configuration.md) - All resource types +- [Patterns](./patterns.md) - Architecture patterns +- [Troubleshooting](./gotchas.md) - Common issues diff --git a/plugins/cloudflare/skills/cloudflare/references/terraform/configuration.md b/plugins/cloudflare/skills/cloudflare/references/terraform/configuration.md new file mode 100644 index 0000000..4b5eeb5 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/terraform/configuration.md @@ -0,0 +1,197 @@ +# Terraform Configuration Reference + +Complete resource configurations for Cloudflare infrastructure. + +## Zone & DNS + +```hcl +# Zone + settings +resource "cloudflare_zone" "example" { account = { id = var.account_id }; name = "example.com"; type = "full" } +resource "cloudflare_zone_settings_override" "example" { + zone_id = cloudflare_zone.example.id + settings { ssl = "strict"; always_use_https = "on"; min_tls_version = "1.2"; tls_1_3 = "on"; http3 = "on" } +} + +# DNS records (A, CNAME, MX, TXT) +resource "cloudflare_dns_record" "www" { + zone_id = cloudflare_zone.example.id; name = "www"; content = "192.0.2.1"; type = "A"; proxied = true +} +resource "cloudflare_dns_record" "mx" { + for_each = { "10" = "mail1.example.com", "20" = "mail2.example.com" } + zone_id = cloudflare_zone.example.id; name = "@"; content = each.value; type = "MX"; priority = each.key +} +``` + +## Workers + +### Simple Pattern (Legacy - Still Works) + +```hcl +resource "cloudflare_workers_script" "api" { + account_id = var.account_id; name = "api-worker"; content = file("worker.js") + module = true; compatibility_date = "2025-01-01" + kv_namespace_binding { name = "KV"; namespace_id = cloudflare_workers_kv_namespace.cache.id } + r2_bucket_binding { name = "BUCKET"; bucket_name = cloudflare_r2_bucket.assets.name } + d1_database_binding { name = "DB"; database_id = cloudflare_d1_database.app.id } + secret_text_binding { name = "SECRET"; text = var.secret } +} +``` + +### Gradual Rollouts (Recommended for Production) + +```hcl +resource "cloudflare_worker" "api" { account_id = var.account_id; name = "api-worker" } +resource "cloudflare_worker_version" "api_v1" { + account_id = var.account_id; worker_name = cloudflare_worker.api.name + content = file("worker.js"); content_sha256 = filesha256("worker.js") + compatibility_date = "2025-01-01" + bindings { + kv_namespace { name = "KV"; namespace_id = cloudflare_workers_kv_namespace.cache.id } + r2_bucket { name = "BUCKET"; bucket_name = cloudflare_r2_bucket.assets.name } + } +} +resource "cloudflare_workers_deployment" "api" { + account_id = var.account_id; worker_name = cloudflare_worker.api.name + versions { version_id = cloudflare_worker_version.api_v1.id; percentage = 100 } +} +``` + +### Worker Binding Types (v5) + +| Binding | Attribute | Example | +|---------|-----------|---------| +| KV | `kv_namespace_binding` | `{ name = "KV", namespace_id = "..." }` | +| R2 | `r2_bucket_binding` | `{ name = "BUCKET", bucket_name = "..." }` | +| D1 | `d1_database_binding` | `{ name = "DB", database_id = "..." }` | +| Service | `service_binding` | `{ name = "AUTH", service = "auth-worker" }` | +| Secret | `secret_text_binding` | `{ name = "API_KEY", text = "..." }` | +| Queue | `queue_binding` | `{ name = "QUEUE", queue_name = "..." }` | +| Vectorize | `vectorize_binding` | `{ name = "INDEX", index_name = "..." }` | +| Hyperdrive | `hyperdrive_binding` | `{ name = "DB", id = "..." }` | +| AI | `ai_binding` | `{ name = "AI" }` | +| Browser | `browser_binding` | `{ name = "BROWSER" }` | +| Analytics | `analytics_engine_binding` | `{ name = "ANALYTICS", dataset = "..." }` | +| mTLS | `mtls_certificate_binding` | `{ name = "CERT", certificate_id = "..." }` | + +### Routes & Triggers + +```hcl +resource "cloudflare_worker_route" "api" { + zone_id = cloudflare_zone.example.id; pattern = "api.example.com/*" + script_name = cloudflare_workers_script.api.name +} +resource "cloudflare_worker_cron_trigger" "task" { + account_id = var.account_id; script_name = cloudflare_workers_script.api.name + schedules = ["*/5 * * * *"] +} +``` + +## Storage (KV, R2, D1) + +```hcl +# KV +resource "cloudflare_workers_kv_namespace" "cache" { account_id = var.account_id; title = "cache" } +resource "cloudflare_workers_kv" "config" { + account_id = var.account_id; namespace_id = cloudflare_workers_kv_namespace.cache.id + key_name = "config"; value = jsonencode({ version = "1.0" }) +} + +# R2 +resource "cloudflare_r2_bucket" "assets" { account_id = var.account_id; name = "assets"; location = "WNAM" } + +# D1 (migrations via wrangler) & Queues +resource "cloudflare_d1_database" "app" { account_id = var.account_id; name = "app-db" } +resource "cloudflare_queue" "events" { account_id = var.account_id; name = "events-queue" } +``` + +## Pages + +```hcl +resource "cloudflare_pages_project" "site" { + account_id = var.account_id; name = "site"; production_branch = "main" + deployment_configs { + production { + compatibility_date = "2025-01-01" + environment_variables = { NODE_ENV = "production" } + kv_namespaces = { KV = cloudflare_workers_kv_namespace.cache.id } + d1_databases = { DB = cloudflare_d1_database.app.id } + } + } + build_config { build_command = "npm run build"; destination_dir = "dist" } + source { type = "github"; config { owner = "org"; repo_name = "site"; production_branch = "main" }} +} + +resource "cloudflare_pages_domain" "custom" { + account_id = var.account_id; project_name = cloudflare_pages_project.site.name; domain = "site.example.com" +} +``` + +## Rulesets (WAF, Redirects, Cache) + +```hcl +# WAF +resource "cloudflare_ruleset" "waf" { + zone_id = cloudflare_zone.example.id; name = "WAF"; kind = "zone"; phase = "http_request_firewall_custom" + rules { action = "block"; enabled = true; expression = "(cf.client.bot) and not (cf.verified_bot)" } +} + +# Redirects +resource "cloudflare_ruleset" "redirects" { + zone_id = cloudflare_zone.example.id; name = "Redirects"; kind = "zone"; phase = "http_request_dynamic_redirect" + rules { + action = "redirect"; enabled = true; expression = "(http.request.uri.path eq \"/old\")" + action_parameters { from_value { status_code = 301; target_url { value = "https://example.com/new" }}} + } +} + +# Cache rules +resource "cloudflare_ruleset" "cache" { + zone_id = cloudflare_zone.example.id; name = "Cache"; kind = "zone"; phase = "http_request_cache_settings" + rules { + action = "set_cache_settings"; enabled = true; expression = "(http.request.uri.path matches \"\\.(jpg|png|css|js)$\")" + action_parameters { cache = true; edge_ttl { mode = "override_origin"; default = 86400 }} + } +} +``` + +## Load Balancers + +```hcl +resource "cloudflare_load_balancer_monitor" "http" { + account_id = var.account_id; type = "http"; path = "/health"; interval = 60; timeout = 5 +} +resource "cloudflare_load_balancer_pool" "api" { + account_id = var.account_id; name = "api-pool"; monitor = cloudflare_load_balancer_monitor.http.id + origins { name = "api-1"; address = "192.0.2.1" } + origins { name = "api-2"; address = "192.0.2.2" } +} +resource "cloudflare_load_balancer" "api" { + zone_id = cloudflare_zone.example.id; name = "api.example.com" + default_pool_ids = [cloudflare_load_balancer_pool.api.id]; steering_policy = "geo" +} +``` + +## Access (Zero Trust) + +```hcl +resource "cloudflare_access_application" "admin" { + account_id = var.account_id; name = "Admin"; domain = "admin.example.com"; type = "self_hosted" + session_duration = "24h"; allowed_idps = [cloudflare_access_identity_provider.github.id] +} +resource "cloudflare_access_policy" "allow" { + account_id = var.account_id; application_id = cloudflare_access_application.admin.id + name = "Allow"; decision = "allow"; precedence = 1 + include { email = ["admin@example.com"] } +} +resource "cloudflare_access_identity_provider" "github" { + account_id = var.account_id; name = "GitHub"; type = "github" + config { client_id = var.github_id; client_secret = var.github_secret } +} +``` + +## See Also + +- [README](./README.md) - Provider setup +- [API](./api.md) - Data sources +- [Patterns](./patterns.md) - Use cases +- [Troubleshooting](./gotchas.md) - Issues diff --git a/plugins/cloudflare/skills/cloudflare/references/terraform/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/terraform/gotchas.md new file mode 100644 index 0000000..eb4731d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/terraform/gotchas.md @@ -0,0 +1,150 @@ +# Terraform Troubleshooting & Best Practices + +Common issues, security considerations, and best practices. + +## State Drift Issues + +Some resources have known state drift. Add lifecycle blocks to prevent perpetual diffs: + +| Resource | Drift Attributes | Workaround | +|----------|------------------|------------| +| `cloudflare_pages_project` | `deployment_configs.*` | `ignore_changes = [deployment_configs]` | +| `cloudflare_workers_script` | secrets returned as REDACTED | `ignore_changes = [secret_text_binding]` | +| `cloudflare_load_balancer` | `adaptive_routing`, `random_steering` | `ignore_changes = [adaptive_routing, random_steering]` | +| `cloudflare_workers_kv` | special chars in keys (< 5.16.0) | Upgrade to 5.16.0+ | + +```hcl +# Example: Ignore secret drift +resource "cloudflare_workers_script" "api" { + account_id = var.account_id + name = "api-worker" + content = file("worker.js") + secret_text_binding { name = "API_KEY"; text = var.api_key } + + lifecycle { + ignore_changes = [secret_text_binding] + } +} +``` + +## v5 Breaking Changes + +Provider v5 is current (auto-generated from OpenAPI). v4→v5 has breaking changes: + +**Resource Renames:** + +| v4 Resource | v5 Resource | Notes | +|-------------|-------------|-------| +| `cloudflare_record` | `cloudflare_dns_record` | | +| `cloudflare_worker_script` | `cloudflare_workers_script` | Note: plural | +| `cloudflare_worker_*` | `cloudflare_workers_*` | All worker resources | +| `cloudflare_access_*` | `cloudflare_zero_trust_*` | Access → Zero Trust | + +**Attribute Changes:** + +| v4 Attribute | v5 Attribute | Resources | +|--------------|--------------|-----------| +| `zone` | `name` | zone | +| `account_id` | `account.id` | zone (object syntax) | +| `key` | `key_name` | KV | +| `location_hint` | `location` | R2 | + +**State Migration:** + +```bash +# Rename resources in state after v5 upgrade +terraform state mv cloudflare_record.example cloudflare_dns_record.example +terraform state mv cloudflare_worker_script.api cloudflare_workers_script.api +``` + +## Resource-Specific Gotchas + +### R2 Location Case Sensitivity + +**Problem:** Terraform creates R2 bucket but fails on subsequent applies +**Cause:** Location must be UPPERCASE +**Solution:** Use `WNAM`, `ENAM`, `WEUR`, `EEUR`, `APAC` (not `wnam`, `enam`, etc.) + +```hcl +resource "cloudflare_r2_bucket" "assets" { + account_id = var.account_id + name = "assets" + location = "WNAM" # UPPERCASE required +} +``` + +### KV Special Characters (< 5.16.0) + +**Problem:** Keys with `+`, `#`, `%` cause encoding issues +**Cause:** URL encoding bug in provider < 5.16.0 +**Solution:** Upgrade to 5.16.0+ or avoid special chars in keys + +### D1 Migrations + +**Problem:** Terraform creates database but schema is empty +**Cause:** Terraform only creates D1 resource, not schema +**Solution:** Run migrations via wrangler after Terraform apply + +```bash +# After terraform apply +wrangler d1 migrations apply +``` + +### Worker Script Size Limit + +**Problem:** Worker deployment fails with "script too large" +**Cause:** Worker script + dependencies exceed 10 MB limit +**Solution:** Use code splitting, external dependencies, or minification + +### Pages Project Drift + +**Problem:** Pages project shows perpetual diff on `deployment_configs` +**Cause:** Cloudflare API adds default values not in Terraform state +**Solution:** Add lifecycle ignore block (see State Drift table above) + +## Common Errors + +### "Error: couldn't find resource" + +**Cause:** Resource was deleted outside Terraform +**Solution:** Import resource back into state with `terraform import cloudflare_zone.example ` or remove from state with `terraform state rm cloudflare_zone.example` + +### "409 Conflict on worker deployment" + +**Cause:** Worker being deployed by both Terraform and wrangler simultaneously +**Solution:** Choose one deployment method; if using Terraform, remove wrangler deployments + +### "DNS record already exists" + +**Cause:** Existing DNS record not imported into Terraform state +**Solution:** Find record ID in Cloudflare dashboard and import with `terraform import cloudflare_dns_record.example /` + +### "Invalid provider configuration" + +**Cause:** API token missing, invalid, or lacking required permissions +**Solution:** Set `CLOUDFLARE_API_TOKEN` environment variable or check token permissions in dashboard + +### "State locking errors" + +**Cause:** Multiple concurrent Terraform runs or stale lock from crashed process +**Solution:** Remove stale lock with `terraform force-unlock ` (use with caution) + +## Limits + +| Resource | Limit | Notes | +|----------|-------|-------| +| API token rate limit | Varies by plan | Use `api_client_logging = true` to debug +| Worker script size | 10 MB | Includes all dependencies +| KV keys per namespace | Unlimited | Pay per operation +| R2 storage | Unlimited | Pay per GB +| D1 databases | 50,000 per account | Free tier: 10 +| Pages projects | 500 per account | 100 for free accounts +| DNS records | 3,500 per zone | Free plan + +## See Also + +- [README](./README.md) - Provider setup +- [Configuration](./configuration.md) - Resources +- [API](./api.md) - Data sources +- [Patterns](./patterns.md) - Use cases +- Provider docs: https://registry.terraform.io/providers/cloudflare/cloudflare/latest/docs diff --git a/plugins/cloudflare/skills/cloudflare/references/terraform/patterns.md b/plugins/cloudflare/skills/cloudflare/references/terraform/patterns.md new file mode 100644 index 0000000..aea3a96 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/terraform/patterns.md @@ -0,0 +1,174 @@ +# Terraform Patterns & Use Cases + +Architecture patterns, multi-environment setups, and real-world use cases. + +## Recommended Directory Structure + +``` +terraform/ +├── environments/ +│ ├── production/ +│ │ ├── main.tf +│ │ └── terraform.tfvars +│ └── staging/ +│ ├── main.tf +│ └── terraform.tfvars +├── modules/ +│ ├── zone/ +│ ├── worker/ +│ └── dns/ +└── shared/ # Shared resources across envs + └── main.tf +``` + +**Note:** Cloudflare recommends avoiding modules for provider resources due to v5 auto-generation complexity. Prefer environment directories + shared state instead. + +## Multi-Environment Setup + +```hcl +# Directory: environments/{production,staging}/main.tf + modules/{zone,worker,pages} +module "zone" { + source = "../../modules/zone"; account_id = var.account_id; zone_name = "example.com"; environment = "production" +} +module "api_worker" { + source = "../../modules/worker"; account_id = var.account_id; zone_id = module.zone.zone_id + name = "api-worker-prod"; script = file("../../workers/api.js"); environment = "production" +} +``` + +## R2 State Backend + +```hcl +terraform { + backend "s3" { + bucket = "terraform-state" + key = "cloudflare.tfstate" + region = "auto" + endpoints = { s3 = "https://.r2.cloudflarestorage.com" } + skip_credentials_validation = true + skip_region_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true + skip_s3_checksum = true + } +} +``` + +## Worker with All Bindings + +```hcl +locals { worker_name = "full-stack-worker" } +resource "cloudflare_workers_kv_namespace" "app" { account_id = var.account_id; title = "${local.worker_name}-kv" } +resource "cloudflare_r2_bucket" "app" { account_id = var.account_id; name = "${local.worker_name}-bucket" } +resource "cloudflare_d1_database" "app" { account_id = var.account_id; name = "${local.worker_name}-db" } + +resource "cloudflare_worker_script" "app" { + account_id = var.account_id; name = local.worker_name; content = file("worker.js"); module = true + compatibility_date = "2025-01-01" + kv_namespace_binding { name = "KV"; namespace_id = cloudflare_workers_kv_namespace.app.id } + r2_bucket_binding { name = "BUCKET"; bucket_name = cloudflare_r2_bucket.app.name } + d1_database_binding { name = "DB"; database_id = cloudflare_d1_database.app.id } + secret_text_binding { name = "API_KEY"; text = var.api_key } +} +``` + +## Wrangler Integration + +**CRITICAL**: Wrangler and Terraform must NOT manage same resources. + +**Terraform**: Zones, DNS, security rules, Access, load balancers, worker deployments (CI/CD), KV/R2/D1 resource creation +**Wrangler**: Local dev (`wrangler dev`), manual deploys, D1 migrations, KV bulk ops, log streaming (`wrangler tail`) + +### CI/CD Pattern + +```hcl +# Terraform creates infrastructure +resource "cloudflare_workers_kv_namespace" "app" { account_id = var.account_id; title = "app-kv" } +resource "cloudflare_d1_database" "app" { account_id = var.account_id; name = "app-db" } +output "kv_namespace_id" { value = cloudflare_workers_kv_namespace.app.id } +output "d1_database_id" { value = cloudflare_d1_database.app.id } +``` + +```yaml +# GitHub Actions: terraform apply → envsubst wrangler.jsonc.template → wrangler deploy +- run: terraform apply -auto-approve +- run: | + export KV_NAMESPACE_ID=$(terraform output -raw kv_namespace_id) + envsubst < wrangler.jsonc.template > wrangler.jsonc +- run: wrangler deploy +``` + +## Use Cases + +### Static Site + API Worker + +```hcl +resource "cloudflare_pages_project" "frontend" { + account_id = var.account_id; name = "frontend"; production_branch = "main" + build_config { build_command = "npm run build"; destination_dir = "dist" } +} +resource "cloudflare_worker_script" "api" { + account_id = var.account_id; name = "api"; content = file("api-worker.js") + d1_database_binding { name = "DB"; database_id = cloudflare_d1_database.api_db.id } +} +resource "cloudflare_dns_record" "frontend" { + zone_id = cloudflare_zone.main.id; name = "app"; content = cloudflare_pages_project.frontend.subdomain; type = "CNAME"; proxied = true +} +resource "cloudflare_worker_route" "api" { + zone_id = cloudflare_zone.main.id; pattern = "api.example.com/*"; script_name = cloudflare_worker_script.api.name +} +``` + +### Multi-Region Load Balancing + +```hcl +resource "cloudflare_load_balancer_pool" "us" { + account_id = var.account_id; name = "us-pool"; monitor = cloudflare_load_balancer_monitor.http.id + origins { name = "us-east"; address = var.us_east_ip } +} +resource "cloudflare_load_balancer_pool" "eu" { + account_id = var.account_id; name = "eu-pool"; monitor = cloudflare_load_balancer_monitor.http.id + origins { name = "eu-west"; address = var.eu_west_ip } +} +resource "cloudflare_load_balancer" "global" { + zone_id = cloudflare_zone.main.id; name = "api.example.com"; steering_policy = "geo" + default_pool_ids = [cloudflare_load_balancer_pool.us.id] + region_pools { region = "WNAM"; pool_ids = [cloudflare_load_balancer_pool.us.id] } + region_pools { region = "WEU"; pool_ids = [cloudflare_load_balancer_pool.eu.id] } +} +``` + +### Secure Admin with Access + +```hcl +resource "cloudflare_pages_project" "admin" { account_id = var.account_id; name = "admin"; production_branch = "main" } +resource "cloudflare_access_application" "admin" { + account_id = var.account_id; name = "Admin"; domain = "admin.example.com"; type = "self_hosted"; session_duration = "24h" + allowed_idps = [cloudflare_access_identity_provider.google.id] +} +resource "cloudflare_access_policy" "allow" { + account_id = var.account_id; application_id = cloudflare_access_application.admin.id + name = "Allow admins"; decision = "allow"; precedence = 1; include { email = var.admin_emails } +} +``` + +### Reusable Module + +```hcl +# modules/cloudflare-zone/main.tf +variable "account_id" { type = string }; variable "domain" { type = string }; variable "ssl_mode" { default = "strict" } +resource "cloudflare_zone" "main" { account = { id = var.account_id }; name = var.domain } +resource "cloudflare_zone_settings_override" "main" { + zone_id = cloudflare_zone.main.id; settings { ssl = var.ssl_mode; always_use_https = "on" } +} +output "zone_id" { value = cloudflare_zone.main.id } + +# Usage: module "prod" { source = "./modules/cloudflare-zone"; account_id = var.account_id; domain = "example.com" } +``` + +## See Also + +- [README](./README.md) - Provider setup +- [Configuration Reference](./configuration.md) - All resource types +- [API Reference](./api.md) - Data sources +- [Troubleshooting](./gotchas.md) - Best practices, common issues diff --git a/plugins/cloudflare/skills/cloudflare/references/tunnel/README.md b/plugins/cloudflare/skills/cloudflare/references/tunnel/README.md new file mode 100644 index 0000000..f70a668 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tunnel/README.md @@ -0,0 +1,129 @@ +# Cloudflare Tunnel + +Secure outbound-only connections between infrastructure and Cloudflare's global network. + +## Overview + +Cloudflare Tunnel (formerly Argo Tunnel) enables: +- **Outbound-only connections** - No inbound ports or firewall changes +- **Public hostname routing** - Expose local services to internet +- **Private network access** - Connect internal networks via WARP +- **Zero Trust integration** - Built-in access policies + +**Architecture**: Tunnel (persistent object) → Replica (`cloudflared` process) → Origin services + +**Terminology:** +- **Tunnel**: Named persistent object with UUID +- **Replica**: Individual `cloudflared` process connected to tunnel +- **Config Source**: Where ingress rules stored (local file vs Cloudflare dashboard) +- **Connector**: Legacy term for replica + +## Quick Start + +### Local Config +```bash +# Install cloudflared +brew install cloudflared # macOS + +# Authenticate +cloudflared tunnel login + +# Create tunnel +cloudflared tunnel create my-tunnel + +# Route DNS +cloudflared tunnel route dns my-tunnel app.example.com + +# Run tunnel +cloudflared tunnel run my-tunnel +``` + +### Dashboard Config (Recommended) +1. **Zero Trust** > **Networks** > **Tunnels** > **Create** +2. Name tunnel, copy token +3. Configure routes in dashboard +4. Run: `cloudflared tunnel --no-autoupdate run --token ` + +## Decision Tree + +**Choose config source:** +``` +Need centralized config updates? +├─ Yes → Token-based (dashboard config) +└─ No → Local config file + +Multiple environments (dev/staging/prod)? +├─ Yes → Local config (version controlled) +└─ No → Either works + +Need firewall approval? +└─ See networking.md first +``` + +## Core Commands + +```bash +# Tunnel lifecycle +cloudflared tunnel create +cloudflared tunnel list +cloudflared tunnel info +cloudflared tunnel delete + +# DNS routing +cloudflared tunnel route dns +cloudflared tunnel route list + +# Private network +cloudflared tunnel route ip add 10.0.0.0/8 + +# Run tunnel +cloudflared tunnel run +``` + +## Configuration Example + +```yaml +# ~/.cloudflared/config.yml +tunnel: 6ff42ae2-765d-4adf-8112-31c55c1551ef +credentials-file: /root/.cloudflared/6ff42ae2-765d-4adf-8112-31c55c1551ef.json + +ingress: + - hostname: app.example.com + service: http://localhost:8000 + - hostname: api.example.com + service: https://localhost:8443 + originRequest: + noTLSVerify: true + - service: http_status:404 +``` + +## Reading Order + +**New to Cloudflare Tunnel:** +1. This README (overview, quick start) +2. [networking.md](./networking.md) - Firewall rules, connectivity pre-checks +3. [configuration.md](./configuration.md) - Config file options, ingress rules +4. [patterns.md](./patterns.md) - Docker, Kubernetes, production deployment +5. [gotchas.md](./gotchas.md) - Troubleshooting, best practices + +**Enterprise deployment:** +1. [networking.md](./networking.md) - Corporate firewall requirements +2. [gotchas.md](./gotchas.md) - HA setup, security best practices +3. [patterns.md](./patterns.md) - Kubernetes, rolling updates + +**Programmatic control:** +1. [api.md](./api.md) - REST API, TypeScript SDK + +## In This Reference + +- [networking.md](./networking.md) - Firewall rules, ports, connectivity pre-checks +- [configuration.md](./configuration.md) - Config file options, ingress rules, TLS settings +- [api.md](./api.md) - REST API, TypeScript SDK, token-based tunnels +- [patterns.md](./patterns.md) - Docker, Kubernetes, Terraform, HA, use cases +- [gotchas.md](./gotchas.md) - Troubleshooting, limitations, best practices + +## See Also + +- [workers](../workers/) - Workers with Tunnel integration +- [access](../access/) - Zero Trust access policies +- [warp](../warp/) - WARP client for private networks diff --git a/plugins/cloudflare/skills/cloudflare/references/tunnel/api.md b/plugins/cloudflare/skills/cloudflare/references/tunnel/api.md new file mode 100644 index 0000000..faa3013 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tunnel/api.md @@ -0,0 +1,193 @@ +# Tunnel API + +## Cloudflare API Access + +**Base URL**: `https://api.cloudflare.com/client/v4` + +**Authentication**: +```bash +Authorization: Bearer ${CF_API_TOKEN} +``` + +## TypeScript SDK + +Install: `npm install cloudflare` + +```typescript +import Cloudflare from 'cloudflare'; + +const cf = new Cloudflare({ + apiToken: process.env.CF_API_TOKEN, +}); + +const accountId = process.env.CF_ACCOUNT_ID; +``` + +## Create Tunnel + +### cURL +```bash +curl -X POST "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + --data '{ + "name": "my-tunnel", + "tunnel_secret": "" + }' +``` + +### TypeScript +```typescript +const tunnel = await cf.zeroTrust.tunnels.create({ + account_id: accountId, + name: 'my-tunnel', + tunnel_secret: Buffer.from(crypto.randomBytes(32)).toString('base64'), +}); + +console.log(`Tunnel ID: ${tunnel.id}`); +``` + +## List Tunnels + +### cURL +```bash +curl -X GET "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" +``` + +### TypeScript +```typescript +const tunnels = await cf.zeroTrust.tunnels.list({ + account_id: accountId, +}); + +for (const tunnel of tunnels.result) { + console.log(`${tunnel.name}: ${tunnel.id}`); +} +``` + +## Get Tunnel Info + +### cURL +```bash +curl -X GET "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" +``` + +### TypeScript +```typescript +const tunnel = await cf.zeroTrust.tunnels.get(tunnelId, { + account_id: accountId, +}); + +console.log(`Status: ${tunnel.status}`); +console.log(`Connections: ${tunnel.connections?.length || 0}`); +``` + +## Update Tunnel Config + +### cURL +```bash +curl -X PUT "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}/configurations" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + --data '{ + "config": { + "ingress": [ + {"hostname": "app.example.com", "service": "http://localhost:8000"}, + {"service": "http_status:404"} + ] + } + }' +``` + +### TypeScript +```typescript +const config = await cf.zeroTrust.tunnels.configurations.update( + tunnelId, + { + account_id: accountId, + config: { + ingress: [ + { hostname: 'app.example.com', service: 'http://localhost:8000' }, + { service: 'http_status:404' }, + ], + }, + } +); +``` + +## Delete Tunnel + +### cURL +```bash +curl -X DELETE "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" +``` + +### TypeScript +```typescript +await cf.zeroTrust.tunnels.delete(tunnelId, { + account_id: accountId, +}); +``` + +## Token-Based Tunnels (Config Source: Cloudflare) + +Token-based tunnels store config in Cloudflare dashboard instead of local files. + +### Via Dashboard +1. **Zero Trust** > **Networks** > **Tunnels** +2. **Create a tunnel** > **Cloudflared** +3. Configure routes in dashboard +4. Copy token +5. Run on origin: +```bash +cloudflared service install +``` + +### Via Token +```bash +# Run with token (no config file needed) +cloudflared tunnel --no-autoupdate run --token ${TUNNEL_TOKEN} + +# Docker +docker run cloudflare/cloudflared:latest tunnel --no-autoupdate run --token ${TUNNEL_TOKEN} +``` + +### Get Tunnel Token (TypeScript) +```typescript +// Get tunnel to retrieve token +const tunnel = await cf.zeroTrust.tunnels.get(tunnelId, { + account_id: accountId, +}); + +// Token available in tunnel.token (only for config source: cloudflare) +const token = tunnel.token; +``` + +## DNS Routes API + +```bash +# Create DNS route +curl -X POST "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}/connections" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + --data '{"hostname": "app.example.com"}' + +# Delete route +curl -X DELETE "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}/connections/{route_id}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" +``` + +## Private Network Routes API + +```bash +# Add IP route +curl -X POST "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}/routes" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + --data '{"ip_network": "10.0.0.0/8"}' + +# List IP routes +curl -X GET "https://api.cloudflare.com/client/v4/accounts/{account_id}/tunnels/{tunnel_id}/routes" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/tunnel/configuration.md b/plugins/cloudflare/skills/cloudflare/references/tunnel/configuration.md new file mode 100644 index 0000000..32b7050 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tunnel/configuration.md @@ -0,0 +1,157 @@ +# Tunnel Configuration + +## Config Source + +Tunnels use one of two config sources: + +| Config Source | Storage | Updates | Use Case | +|---------------|---------|---------|----------| +| Local | `config.yml` file | Edit file, restart | Dev, multi-env, version control | +| Cloudflare | Dashboard/API | Instant, no restart | Production, centralized management | + +**Token-based tunnels** = config source: Cloudflare +**Locally-managed tunnels** = config source: local + +## Config File Location + +``` +~/.cloudflared/config.yml # User config +/etc/cloudflared/config.yml # System-wide (Linux) +``` + +## Basic Structure + +```yaml +tunnel: +credentials-file: /path/to/.json + +ingress: + - hostname: app.example.com + service: http://localhost:8000 + - service: http_status:404 # Required catch-all +``` + +## Ingress Rules + +Rules evaluated **top to bottom**, first match wins. + +```yaml +ingress: + # Exact hostname + path regex + - hostname: static.example.com + path: \.(jpg|png|css|js)$ + service: https://localhost:8001 + + # Wildcard hostname + - hostname: "*.example.com" + service: https://localhost:8002 + + # Path only (all hostnames) + - path: /api/.* + service: http://localhost:9000 + + # Catch-all (required) + - service: http_status:404 +``` + +**Validation**: +```bash +cloudflared tunnel ingress validate +cloudflared tunnel ingress rule https://foo.example.com +``` + +## Service Types + +| Protocol | Format | Client Requirement | +|----------|--------|-------------------| +| HTTP | `http://localhost:8000` | Browser | +| HTTPS | `https://localhost:8443` | Browser | +| TCP | `tcp://localhost:2222` | `cloudflared access tcp` | +| SSH | `ssh://localhost:22` | `cloudflared access ssh` | +| RDP | `rdp://localhost:3389` | `cloudflared access rdp` | +| Unix | `unix:/path/to/socket` | Browser | +| Test | `hello_world` | Browser | + +## Origin Configuration + +### Connection Settings +```yaml +originRequest: + connectTimeout: 30s + tlsTimeout: 10s + tcpKeepAlive: 30s + keepAliveTimeout: 90s + keepAliveConnections: 100 +``` + +### TLS Settings +```yaml +originRequest: + noTLSVerify: true # Disable cert verification + originServerName: "app.internal" # Override SNI + caPool: /path/to/ca.pem # Custom CA +``` + +### HTTP Settings +```yaml +originRequest: + disableChunkedEncoding: true + httpHostHeader: "app.internal" + http2Origin: true +``` + +## Private Network Mode + +```yaml +tunnel: +credentials-file: /path/to/creds.json + +warp-routing: + enabled: true +``` + +```bash +cloudflared tunnel route ip add 10.0.0.0/8 my-tunnel +cloudflared tunnel route ip add 192.168.1.100/32 my-tunnel +``` + +## Config Source Comparison + +### Local Config +```yaml +# config.yml +tunnel: +credentials-file: /path/to/.json + +ingress: + - hostname: app.example.com + service: http://localhost:8000 + - service: http_status:404 +``` + +```bash +cloudflared tunnel run my-tunnel +``` + +**Pros:** Version control, multi-environment, offline edits +**Cons:** Requires file distribution, manual restarts + +### Cloudflare Config (Token-Based) +```bash +# No config file needed +cloudflared tunnel --no-autoupdate run --token +``` + +Configure routes in dashboard: **Zero Trust** > **Networks** > **Tunnels** > [Tunnel] > **Public Hostname** + +**Pros:** Centralized updates, no file management, instant route changes +**Cons:** Requires dashboard/API access, less portable + +## Environment Variables + +```bash +TUNNEL_TOKEN= # Token for config source: cloudflare +TUNNEL_ORIGIN_CERT=/path/to/cert.pem # Override cert path (local config) +NO_AUTOUPDATE=true # Disable auto-updates +TUNNEL_LOGLEVEL=debug # Log level +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/tunnel/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/tunnel/gotchas.md new file mode 100644 index 0000000..f368856 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tunnel/gotchas.md @@ -0,0 +1,147 @@ +# Tunnel Gotchas + +## Common Errors + +### "Error 1016 (Origin DNS Error)" + +**Cause:** Tunnel not running or not connected +**Solution:** +```bash +cloudflared tunnel info my-tunnel # Check status +ps aux | grep cloudflared # Verify running +journalctl -u cloudflared -n 100 # Check logs +``` + +### "Self-signed certificate rejected" + +**Cause:** Origin using self-signed certificate +**Solution:** +```yaml +originRequest: + noTLSVerify: true # Dev only + caPool: /path/to/ca.pem # Custom CA +``` + +### "Connection timeout" + +**Cause:** Origin slow to respond or timeout settings too low +**Solution:** +```yaml +originRequest: + connectTimeout: 60s + tlsTimeout: 20s + keepAliveTimeout: 120s +``` + +### "Tunnel not starting" + +**Cause:** Invalid config, missing credentials, or tunnel doesn't exist +**Solution:** +```bash +cloudflared tunnel ingress validate # Validate config +ls -la ~/.cloudflared/*.json # Verify credentials +cloudflared tunnel list # Verify tunnel exists +``` + +### "Connection already registered" + +**Cause:** Multiple replicas with same connector ID or stale connection +**Solution:** +```bash +# Check active connections +cloudflared tunnel info my-tunnel + +# Wait 60s for stale connection cleanup, or restart with new connector ID +cloudflared tunnel run my-tunnel +``` + +### "Tunnel credentials rotated but connections fail" + +**Cause:** Old cloudflared processes using expired credentials +**Solution:** +```bash +# Stop all cloudflared processes +pkill cloudflared + +# Verify stopped +ps aux | grep cloudflared + +# Restart with new credentials +cloudflared tunnel run my-tunnel +``` + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Free tier | Unlimited tunnels | Unlimited traffic | +| Tunnel replicas | 1000 per tunnel | Max concurrent | +| Connection duration | No hard limit | Hours to days | +| Long-lived connections | May drop during updates | WebSocket, SSH, UDP | +| Replica registration | ~5s TTL | Old replica dropped after 5s no heartbeat | +| Token rotation grace | 24 hours | Old tokens work during grace period | + +## Best Practices + +### Security +1. Use token-based tunnels (config source: cloudflare) for centralized control +2. Enable Access policies for sensitive services +3. Rotate tunnel credentials regularly +4. After rotation: stop all old cloudflared processes within 24h grace period +5. Verify TLS certs (`noTLSVerify: false`) +6. Restrict `bastion` service type + +### Performance +1. Run multiple replicas for HA (2-4 typical, load balanced automatically) +2. Replicas share same tunnel UUID, get unique connector IDs +3. Place `cloudflared` close to origin (same network) +4. Use HTTP/2 for gRPC (`http2Origin: true`) +5. Tune keepalive for long-lived connections +6. Monitor connection counts + +### Configuration +1. Use environment variables for secrets +2. Version control config files +3. Validate before deploying (`cloudflared tunnel ingress validate`) +4. Test rules (`cloudflared tunnel ingress rule `) +5. Document rule order (first match wins) + +### Operations +1. Monitor tunnel health in dashboard (shows active replicas) +2. Set up disconnect alerts (when replica count drops to 0) +3. Graceful shutdown for config updates +4. Update replicas in rolling fashion (update 1, wait, update next) +5. Keep `cloudflared` updated (1 year support window) +6. Use `--no-autoupdate` in prod; control updates manually + +## Debug Mode + +```bash +cloudflared tunnel --loglevel debug run my-tunnel +cloudflared tunnel ingress rule https://app.example.com +``` + +## Migration Strategies + +### From Ngrok +```yaml +# Ngrok: ngrok http 8000 +# Cloudflare Tunnel: +ingress: + - hostname: app.example.com + service: http://localhost:8000 + - service: http_status:404 +``` + +### From VPN +```yaml +# Replace VPN with private network routing +warp-routing: + enabled: true +``` + +```bash +cloudflared tunnel route ip add 10.0.0.0/8 my-tunnel +``` + +Users install WARP client instead of VPN. diff --git a/plugins/cloudflare/skills/cloudflare/references/tunnel/networking.md b/plugins/cloudflare/skills/cloudflare/references/tunnel/networking.md new file mode 100644 index 0000000..8c3930d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tunnel/networking.md @@ -0,0 +1,168 @@ +# Tunnel Networking + +## Connectivity Requirements + +### Outbound Ports + +Cloudflared requires outbound access on: + +| Port | Protocol | Purpose | Required | +|------|----------|---------|----------| +| 7844 | TCP/UDP | Primary tunnel protocol (QUIC) | Yes | +| 443 | TCP | Fallback (HTTP/2) | Yes | + +**Network path:** +``` +cloudflared → edge.argotunnel.com:7844 (preferred) +cloudflared → region.argotunnel.com:443 (fallback) +``` + +### Firewall Rules + +#### Minimal (Production) +```bash +# Outbound only +ALLOW tcp/udp 7844 to *.argotunnel.com +ALLOW tcp 443 to *.argotunnel.com +``` + +#### Full (Recommended) +```bash +# Tunnel connectivity +ALLOW tcp/udp 7844 to *.argotunnel.com +ALLOW tcp 443 to *.argotunnel.com + +# API access (for token-based tunnels) +ALLOW tcp 443 to api.cloudflare.com + +# Updates (optional) +ALLOW tcp 443 to github.com +ALLOW tcp 443 to objects.githubusercontent.com +``` + +### IP Ranges + +Cloudflare Anycast IPs (tunnel endpoints): +``` +# IPv4 +198.41.192.0/24 +198.41.200.0/24 + +# IPv6 +2606:4700::/32 +``` + +**Note:** Use DNS resolution for `*.argotunnel.com` rather than hardcoding IPs. Cloudflare may add edge locations. + +## Pre-Flight Check + +Test connectivity before deploying: + +```bash +# Test DNS resolution +dig edge.argotunnel.com +short + +# Test port 7844 (QUIC/UDP) +nc -zvu edge.argotunnel.com 7844 + +# Test port 443 (HTTP/2 fallback) +nc -zv edge.argotunnel.com 443 + +# Test with cloudflared +cloudflared tunnel --loglevel debug run my-tunnel +# Look for "Registered tunnel connection" +``` + +### Common Connectivity Errors + +| Error | Cause | Solution | +|-------|-------|----------| +| "no such host" | DNS blocked | Allow port 53 UDP/TCP | +| "context deadline exceeded" | Port 7844 blocked | Allow UDP/TCP 7844 | +| "TLS handshake timeout" | Port 443 blocked | Allow TCP 443, disable SSL inspection | + +## Protocol Selection + +Cloudflared automatically selects protocol: + +| Protocol | Port | Priority | Use Case | +|----------|------|----------|----------| +| QUIC | 7844 UDP | 1st (preferred) | Low latency, best performance | +| HTTP/2 | 443 TCP | 2nd (fallback) | QUIC blocked by firewall | + +**Force HTTP/2 fallback:** +```bash +cloudflared tunnel --protocol http2 run my-tunnel +``` + +**Verify active protocol:** +```bash +cloudflared tunnel info my-tunnel +# Shows "connections" with protocol type +``` + +## Private Network Routing + +### WARP Client Requirements + +Users accessing private IPs via WARP need: + +```bash +# Outbound (WARP client) +ALLOW udp 500,4500 to 162.159.*.* (IPsec) +ALLOW udp 2408 to 162.159.*.* (WireGuard) +ALLOW tcp 443 to *.cloudflareclient.com +``` + +### Split Tunnel Configuration + +Route only private networks through tunnel: + +```yaml +# warp-routing config +warp-routing: + enabled: true +``` + +```bash +# Add specific routes +cloudflared tunnel route ip add 10.0.0.0/8 my-tunnel +cloudflared tunnel route ip add 172.16.0.0/12 my-tunnel +cloudflared tunnel route ip add 192.168.0.0/16 my-tunnel +``` + +WARP users can access these IPs without VPN. + +## Network Diagnostics + +### Connection Diagnostics + +```bash +# Check edge selection and connection health +cloudflared tunnel info my-tunnel --output json | jq '.connections[]' + +# Enable metrics endpoint +cloudflared tunnel --metrics localhost:9090 run my-tunnel +curl localhost:9090/metrics | grep cloudflared_tunnel + +# Test latency +curl -w "time_total: %{time_total}\n" -o /dev/null https://myapp.example.com +``` + +## Corporate Network Considerations + +Cloudflared honors proxy environment variables (`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`). + +If corporate proxy intercepts TLS, add corporate root CA to system trust store. + +## Bandwidth and Rate Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Request size | 100 MB | Single HTTP request | +| Upload speed | No hard limit | Governed by network/plan | +| Concurrent connections | 1000 per tunnel | Across all replicas | +| Requests per second | No limit | Subject to DDoS detection | + +**Large file transfers:** +Use R2 or Workers with chunked uploads instead of streaming through tunnel. diff --git a/plugins/cloudflare/skills/cloudflare/references/tunnel/patterns.md b/plugins/cloudflare/skills/cloudflare/references/tunnel/patterns.md new file mode 100644 index 0000000..7ef6d29 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/tunnel/patterns.md @@ -0,0 +1,192 @@ +# Tunnel Patterns + +## Docker Deployment + +### Token-Based (Recommended) +```yaml +services: + cloudflared: + image: cloudflare/cloudflared:latest + command: tunnel --no-autoupdate run --token ${TUNNEL_TOKEN} + restart: unless-stopped +``` + +### Local Config +```yaml +services: + cloudflared: + image: cloudflare/cloudflared:latest + volumes: + - ./config.yml:/etc/cloudflared/config.yml:ro + - ./credentials.json:/etc/cloudflared/credentials.json:ro + command: tunnel run +``` + +## Kubernetes Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudflared +spec: + replicas: 2 + selector: + matchLabels: + app: cloudflared + template: + metadata: + labels: + app: cloudflared + spec: + containers: + - name: cloudflared + image: cloudflare/cloudflared:latest + args: + - tunnel + - --no-autoupdate + - run + - --token + - $(TUNNEL_TOKEN) + env: + - name: TUNNEL_TOKEN + valueFrom: + secretKeyRef: + name: tunnel-credentials + key: token +``` + +## High Availability + +```yaml +# Same config on multiple servers +tunnel: +credentials-file: /path/to/creds.json + +ingress: + - hostname: app.example.com + service: http://localhost:8000 + - service: http_status:404 +``` + +Run same config on multiple machines. Cloudflare automatically load balances. Long-lived connections (WebSocket, SSH) may drop during updates. + +## Use Cases + +### Web Application +```yaml +ingress: + - hostname: myapp.example.com + service: http://localhost:3000 + - service: http_status:404 +``` + +### SSH Access +```yaml +ingress: + - hostname: ssh.example.com + service: ssh://localhost:22 + - service: http_status:404 +``` + +Client: `cloudflared access ssh --hostname ssh.example.com` + +### gRPC Service +```yaml +ingress: + - hostname: grpc.example.com + service: http://localhost:50051 + originRequest: + http2Origin: true + - service: http_status:404 +``` + +## Infrastructure as Code + +### Terraform + +```hcl +resource "random_id" "tunnel_secret" { + byte_length = 32 +} + +resource "cloudflare_tunnel" "app" { + account_id = var.cloudflare_account_id + name = "app-tunnel" + secret = random_id.tunnel_secret.b64_std +} + +resource "cloudflare_tunnel_config" "app" { + account_id = var.cloudflare_account_id + tunnel_id = cloudflare_tunnel.app.id + config { + ingress_rule { + hostname = "app.example.com" + service = "http://localhost:8000" + } + ingress_rule { service = "http_status:404" } + } +} + +resource "cloudflare_record" "app" { + zone_id = var.cloudflare_zone_id + name = "app" + value = cloudflare_tunnel.app.cname + type = "CNAME" + proxied = true +} + +output "tunnel_token" { + value = cloudflare_tunnel.app.tunnel_token + sensitive = true +} +``` + +### Pulumi + +```typescript +import * as cloudflare from "@pulumi/cloudflare"; +import * as random from "@pulumi/random"; + +const secret = new random.RandomId("secret", { byteLength: 32 }); + +const tunnel = new cloudflare.ZeroTrustTunnelCloudflared("tunnel", { + accountId: accountId, + name: "app-tunnel", + secret: secret.b64Std, +}); + +const config = new cloudflare.ZeroTrustTunnelCloudflaredConfig("config", { + accountId: accountId, + tunnelId: tunnel.id, + config: { + ingressRules: [ + { hostname: "app.example.com", service: "http://localhost:8000" }, + { service: "http_status:404" }, + ], + }, +}); + +new cloudflare.Record("dns", { + zoneId: zoneId, + name: "app", + value: tunnel.cname, + type: "CNAME", + proxied: true, +}); +``` + +## Service Installation + +### Linux systemd +```bash +cloudflared service install +systemctl start cloudflared && systemctl enable cloudflared +journalctl -u cloudflared -f # Logs +``` + +### macOS launchd +```bash +sudo cloudflared service install +sudo launchctl start com.cloudflare.cloudflared +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/turn/README.md b/plugins/cloudflare/skills/cloudflare/references/turn/README.md new file mode 100644 index 0000000..cc4b39e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turn/README.md @@ -0,0 +1,82 @@ +# Cloudflare TURN Service + +Expert guidance for implementing Cloudflare TURN Service in WebRTC applications. + +## Overview + +Cloudflare TURN (Traversal Using Relays around NAT) Service is a managed relay service for WebRTC applications. TURN acts as a relay point for traffic between WebRTC clients and SFUs, particularly when direct peer-to-peer communication is obstructed by NATs or firewalls. The service runs on Cloudflare's global anycast network across 310+ cities. + +## Key Characteristics + +- **Anycast Architecture**: Automatically connects clients to the closest Cloudflare location +- **Global Network**: Available across Cloudflare's entire network (excluding China Network) +- **Zero Configuration**: No need to manually select regions or servers +- **Protocol Support**: STUN/TURN over UDP, TCP, and TLS +- **Free Tier**: Free when used with Cloudflare Calls SFU, otherwise $0.05/GB outbound + +## In This Reference + +| File | Purpose | +|------|---------| +| [api.md](./api.md) | Credentials API, TURN key management, types, constraints | +| [configuration.md](./configuration.md) | Worker setup, wrangler.jsonc, env vars, IP allowlisting | +| [patterns.md](./patterns.md) | Implementation patterns, use cases, integration examples | +| [gotchas.md](./gotchas.md) | Troubleshooting, limits, security, common mistakes | + +## Reading Order + +| Task | Files to Read | Est. Tokens | +|------|---------------|-------------| +| Quick start | README only | ~500 | +| Generate credentials | README → api | ~1300 | +| Worker integration | README → configuration → patterns | ~2000 | +| Debug connection | gotchas | ~700 | +| Security review | api → gotchas | ~1500 | +| Enterprise firewall | configuration | ~600 | + +## Service Addresses and Ports + +### STUN over UDP +- **Primary**: `stun.cloudflare.com:3478/udp` +- **Alternate**: `stun.cloudflare.com:53/udp` (blocked by browsers, not recommended) + +### TURN over UDP +- **Primary**: `turn.cloudflare.com:3478/udp` +- **Alternate**: `turn.cloudflare.com:53/udp` (blocked by browsers) + +### TURN over TCP +- **Primary**: `turn.cloudflare.com:3478/tcp` +- **Alternate**: `turn.cloudflare.com:80/tcp` + +### TURN over TLS +- **Primary**: `turn.cloudflare.com:5349/tcp` +- **Alternate**: `turn.cloudflare.com:443/tcp` + +## Quick Start + +1. **Create TURN key via API**: see [api.md#create-turn-key](./api.md#create-turn-key) +2. **Generate credentials**: see [api.md#generate-temporary-credentials](./api.md#generate-temporary-credentials) +3. **Configure Worker**: see [configuration.md#cloudflare-worker-integration](./configuration.md#cloudflare-worker-integration) +4. **Implement client**: see [patterns.md#basic-turn-configuration-browser](./patterns.md#basic-turn-configuration-browser) + +## When to Use TURN + +- **Restrictive NATs**: Symmetric NATs that block direct connections +- **Corporate firewalls**: Environments blocking WebRTC ports +- **Mobile networks**: Carrier-grade NAT scenarios +- **Predictable connectivity**: When reliability > efficiency + +## Related Cloudflare Services + +- **Cloudflare Calls SFU**: Managed Selective Forwarding Unit (TURN free when used with SFU) +- **Cloudflare Stream**: Video streaming with WHIP/WHEP support +- **Cloudflare Workers**: Backend for credential generation +- **Cloudflare KV**: Credential caching +- **Cloudflare Durable Objects**: Session state management + +## Additional Resources + +- [Cloudflare Calls Documentation](https://developers.cloudflare.com/calls/) +- [Cloudflare TURN Service Docs](https://developers.cloudflare.com/realtime/turn/) +- [Cloudflare API Reference](https://developers.cloudflare.com/api/resources/calls/subresources/turn/) +- [Orange Meets (Open Source Example)](https://github.com/cloudflare/orange) diff --git a/plugins/cloudflare/skills/cloudflare/references/turn/api.md b/plugins/cloudflare/skills/cloudflare/references/turn/api.md new file mode 100644 index 0000000..498f5e4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turn/api.md @@ -0,0 +1,239 @@ +# TURN API Reference + +Complete API documentation for Cloudflare TURN service credentials and key management. + +## Authentication + +All endpoints require Cloudflare API token with "Calls Write" permission. + +Base URL: `https://api.cloudflare.com/client/v4` + +## TURN Key Management + +### List TURN Keys + +``` +GET /accounts/{account_id}/calls/turn_keys +``` + +### Get TURN Key Details + +``` +GET /accounts/{account_id}/calls/turn_keys/{key_id} +``` + +### Create TURN Key + +``` +POST /accounts/{account_id}/calls/turn_keys +Content-Type: application/json + +{ + "name": "my-turn-key" +} +``` + +**Response includes**: +- `uid`: Key identifier +- `key`: The actual secret key (only returned on creation—save immediately) +- `name`: Human-readable name +- `created`: ISO 8601 timestamp +- `modified`: ISO 8601 timestamp + +### Update TURN Key + +``` +PUT /accounts/{account_id}/calls/turn_keys/{key_id} +Content-Type: application/json + +{ + "name": "updated-name" +} +``` + +### Delete TURN Key + +``` +DELETE /accounts/{account_id}/calls/turn_keys/{key_id} +``` + +## Generate Temporary Credentials + +``` +POST https://rtc.live.cloudflare.com/v1/turn/keys/{key_id}/credentials/generate +Authorization: Bearer {key_secret} +Content-Type: application/json + +{ + "ttl": 86400 +} +``` + +### Credential Constraints + +| Parameter | Min | Max | Default | Notes | +|-----------|-----|-----|---------|-------| +| ttl | 1 | 172800 (48hrs) | varies | API rejects values >172800 | + +**CRITICAL**: Maximum TTL is 48 hours (172800 seconds). API will reject requests exceeding this limit. + +### Response Schema + +```json +{ + "iceServers": { + "urls": [ + "stun:stun.cloudflare.com:3478", + "turn:turn.cloudflare.com:3478?transport=udp", + "turn:turn.cloudflare.com:3478?transport=tcp", + "turn:turn.cloudflare.com:53?transport=udp", + "turn:turn.cloudflare.com:80?transport=tcp", + "turns:turn.cloudflare.com:5349?transport=tcp", + "turns:turn.cloudflare.com:443?transport=tcp" + ], + "username": "1738035200:user123", + "credential": "base64encodedhmac==" + } +} +``` + +**Port 53 Warning**: Filter port 53 URLs for browser clients—blocked by Chrome/Firefox. See [gotchas.md](./gotchas.md#using-port-53-in-browsers). + +## Revoke Credentials + +``` +POST https://rtc.live.cloudflare.com/v1/turn/keys/{key_id}/credentials/revoke +Authorization: Bearer {key_secret} +Content-Type: application/json + +{ + "username": "1738035200:user123" +} +``` + +**Response**: 204 No Content + +Billing stops immediately. Active connection drops after short delay (~seconds). + +## TypeScript Types + +```typescript +interface CloudflareTURNConfig { + keyId: string; + keySecret: string; + ttl?: number; // Max 172800 (48 hours) +} + +interface TURNCredentialsRequest { + ttl?: number; // Max 172800 seconds +} + +interface TURNCredentialsResponse { + iceServers: { + urls: string[]; + username: string; + credential: string; + }; +} + +interface RTCIceServer { + urls: string | string[]; + username?: string; + credential?: string; + credentialType?: "password"; +} + +interface TURNKeyResponse { + uid: string; + key: string; // Only present on creation + name: string; + created: string; + modified: string; +} +``` + +## Validation Function + +```typescript +function validateRTCIceServer(obj: unknown): obj is RTCIceServer { + if (!obj || typeof obj !== 'object') { + return false; + } + + const server = obj as Record; + + if (typeof server.urls !== 'string' && !Array.isArray(server.urls)) { + return false; + } + + if (server.username && typeof server.username !== 'string') { + return false; + } + + if (server.credential && typeof server.credential !== 'string') { + return false; + } + + return true; +} +``` + +## Type-Safe Credential Generation + +```typescript +async function fetchTURNServers( + config: CloudflareTURNConfig +): Promise { + // Validate TTL constraint + const ttl = config.ttl ?? 3600; + if (ttl > 172800) { + throw new Error('TTL cannot exceed 172800 seconds (48 hours)'); + } + + const response = await fetch( + `https://rtc.live.cloudflare.com/v1/turn/keys/${config.keyId}/credentials/generate`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${config.keySecret}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ ttl }) + } + ); + + if (!response.ok) { + throw new Error(`TURN credential generation failed: ${response.status}`); + } + + const data = await response.json(); + + // Filter port 53 for browser clients + const filteredUrls = data.iceServers.urls.filter( + (url: string) => !url.includes(':53') + ); + + const iceServers = [ + { urls: 'stun:stun.cloudflare.com:3478' }, + { + urls: filteredUrls, + username: data.iceServers.username, + credential: data.iceServers.credential, + credentialType: 'password' as const + } + ]; + + // Validate before returning + if (!iceServers.every(validateRTCIceServer)) { + throw new Error('Invalid ICE server configuration received'); + } + + return iceServers; +} +``` + +## See Also + +- [configuration.md](./configuration.md) - Worker setup, environment variables +- [patterns.md](./patterns.md) - Implementation examples using these APIs +- [gotchas.md](./gotchas.md) - Security best practices, common mistakes diff --git a/plugins/cloudflare/skills/cloudflare/references/turn/configuration.md b/plugins/cloudflare/skills/cloudflare/references/turn/configuration.md new file mode 100644 index 0000000..2d49736 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turn/configuration.md @@ -0,0 +1,179 @@ +# TURN Configuration + +Setup and configuration for Cloudflare TURN service in Workers and applications. + +## Environment Variables + +```bash +# .env +CLOUDFLARE_ACCOUNT_ID=your_account_id +CLOUDFLARE_API_TOKEN=your_api_token +TURN_KEY_ID=your_turn_key_id +TURN_KEY_SECRET=your_turn_key_secret +``` + +Validate with zod: + +```typescript +import { z } from 'zod'; + +const envSchema = z.object({ + CLOUDFLARE_ACCOUNT_ID: z.string().min(1), + CLOUDFLARE_API_TOKEN: z.string().min(1), + TURN_KEY_ID: z.string().min(1), + TURN_KEY_SECRET: z.string().min(1) +}); + +export const config = envSchema.parse(process.env); +``` + +## wrangler.jsonc + +```jsonc +{ + "name": "turn-credentials-api", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", + "vars": { + "TURN_KEY_ID": "your-turn-key-id" // Non-sensitive, can be in vars + }, + "env": { + "production": { + "kv_namespaces": [ + { + "binding": "CREDENTIALS_CACHE", + "id": "your-kv-namespace-id" + } + ] + } + } +} +``` + +**Store secrets separately**: +```bash +wrangler secret put TURN_KEY_SECRET +``` + +## Cloudflare Worker Integration + +### Worker Binding Types + +```typescript +interface Env { + TURN_KEY_ID: string; + TURN_KEY_SECRET: string; + CREDENTIALS_CACHE?: KVNamespace; +} + +export default { + async fetch(request: Request, env: Env): Promise { + // See patterns.md for implementation + } +} +``` + +### Basic Worker Example + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + if (request.url.endsWith('/turn-credentials')) { + // Validate client auth + const authHeader = request.headers.get('Authorization'); + if (!authHeader) { + return new Response('Unauthorized', { status: 401 }); + } + + const response = await fetch( + `https://rtc.live.cloudflare.com/v1/turn/keys/${env.TURN_KEY_ID}/credentials/generate`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${env.TURN_KEY_SECRET}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ ttl: 3600 }) + } + ); + + if (!response.ok) { + return new Response('Failed to generate credentials', { status: 500 }); + } + + const data = await response.json(); + + // Filter port 53 for browser clients + const filteredUrls = data.iceServers.urls.filter( + (url: string) => !url.includes(':53') + ); + + return Response.json({ + iceServers: [ + { urls: 'stun:stun.cloudflare.com:3478' }, + { + urls: filteredUrls, + username: data.iceServers.username, + credential: data.iceServers.credential + } + ] + }); + } + + return new Response('Not found', { status: 404 }); + } +}; +``` + +## IP Allowlisting (Enterprise/Firewall) + +For strict firewalls, allowlist these IPs for `turn.cloudflare.com`: + +| Type | Address | Protocol | +|------|---------|----------| +| IPv4 | 141.101.90.1/32 | All | +| IPv4 | 162.159.207.1/32 | All | +| IPv6 | 2a06:98c1:3200::1/128 | All | +| IPv6 | 2606:4700:48::1/128 | All | + +**IMPORTANT**: These IPs may change with 14-day notice. Monitor DNS: + +```bash +# Check A and AAAA records +dig turn.cloudflare.com A +dig turn.cloudflare.com AAAA +``` + +Set up automated monitoring to detect IP changes and update allowlists within 14 days. + +## IPv6 Support + +- **Client-to-TURN**: Both IPv4 and IPv6 supported +- **Relay addresses**: IPv4 only (no RFC 6156 support) +- **TCP relaying**: Not supported (RFC 6062) + +Clients can connect via IPv6, but relayed traffic uses IPv4 addresses. + +## TLS Configuration + +### Supported TLS Versions +- TLS 1.1 +- TLS 1.2 +- TLS 1.3 + +### Recommended Ciphers (TLS 1.3) +- AEAD-AES128-GCM-SHA256 +- AEAD-AES256-GCM-SHA384 +- AEAD-CHACHA20-POLY1305-SHA256 + +### Recommended Ciphers (TLS 1.2) +- ECDHE-ECDSA-AES128-GCM-SHA256 +- ECDHE-RSA-AES128-GCM-SHA256 +- ECDHE-RSA-AES128-SHA (also TLS 1.1) +- AES128-GCM-SHA256 + +## See Also + +- [api.md](./api.md) - TURN key creation, credential generation API +- [patterns.md](./patterns.md) - Full Worker implementation patterns +- [gotchas.md](./gotchas.md) - Security best practices, troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/turn/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/turn/gotchas.md new file mode 100644 index 0000000..e2d5bd1 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turn/gotchas.md @@ -0,0 +1,231 @@ +# TURN Gotchas & Troubleshooting + +Common mistakes, security best practices, and troubleshooting for Cloudflare TURN. + +## Quick Reference + +| Issue | Solution | Details | +|-------|----------|---------| +| Credentials not working | Check TTL ≤ 48hrs | [See Troubleshooting](#issue-turn-credentials-not-working) | +| Connection drops after ~48hrs | Implement credential refresh | [See Connection Drops](#issue-connection-drops-after-48-hours) | +| Port 53 fails in browser | Filter server-side | [See Port 53](#using-port-53-in-browsers) | +| High packet loss | Check rate limits | [See Rate Limits](#limits-per-turn-allocation) | +| Connection fails after maintenance | Implement ICE restart | [See ICE Restart](#ice-restart-required-scenarios) | + +## Critical Constraints + +| Constraint | Value | Consequence if Violated | +|------------|-------|-------------------------| +| Max credential TTL | 48 hours (172800s) | API rejects request | +| Credential revocation delay | ~seconds | Billing stops immediately, connection drops shortly | +| IP allowlist update window | 14 days (if IPs change) | Connection fails if IPs change | +| Packet rate | 5-10k pps per allocation | Packet drops | +| Data rate | 50-100 Mbps per allocation | Packet drops | +| Unique IP rate | >5 new IPs/sec | Packet drops | + +## Limits Per TURN Allocation + +**Per user** (not account-wide): + +- **IP addresses**: >5 new unique IPs per second +- **Packet rate**: 5-10k packets per second (inbound/outbound) +- **Data rate**: 50-100 Mbps (inbound/outbound) +- **MTU**: No specific limit +- **Burst rates**: Higher than documented + +Exceeding limits results in **packet drops**. + +## Common Mistakes + +### Setting TTL > 48 hours + +```typescript +// ❌ BAD: API will reject +const creds = await generate({ ttl: 604800 }); // 7 days + +// ✅ GOOD: +const creds = await generate({ ttl: 86400 }); // 24 hours +``` + +### Hardcoding IPs without monitoring + +```typescript +// ❌ BAD: IPs can change with 14-day notice +const iceServers = [{ urls: 'turn:141.101.90.1:3478' }]; + +// ✅ GOOD: Use DNS +const iceServers = [{ urls: 'turn:turn.cloudflare.com:3478' }]; +``` + +### Using port 53 in browsers + +```typescript +// ❌ BAD: Blocked by Chrome/Firefox +urls: ['turn:turn.cloudflare.com:53'] + +// ✅ GOOD: Filter port 53 +urls: urls.filter(url => !url.includes(':53')) +``` + +### Not handling credential expiry + +```typescript +// ❌ BAD: Credentials expire but call continues → connection drops +const creds = await fetchCreds(); +const pc = new RTCPeerConnection({ iceServers: creds }); + +// ✅ GOOD: Refresh before expiry +setInterval(() => refreshCredentials(pc), 3000000); // 50 min +``` + +### Missing ICE restart support + +```typescript +// ❌ BAD: No recovery from TURN maintenance +pc.addEventListener('iceconnectionstatechange', () => { + console.log('State changed:', pc.iceConnectionState); +}); + +// ✅ GOOD: Implement ICE restart +pc.addEventListener('iceconnectionstatechange', async () => { + if (pc.iceConnectionState === 'failed') { + await refreshCredentials(pc); + pc.restartIce(); + } +}); +``` + +### Exposing TURN key secret client-side + +```typescript +// ❌ BAD: Secret exposed to client +const secret = 'your-turn-key-secret'; +const response = await fetch(`https://rtc.live.cloudflare.com/v1/turn/...`, { + headers: { 'Authorization': `Bearer ${secret}` } +}); + +// ✅ GOOD: Generate credentials server-side +const response = await fetch('/api/turn-credentials'); +``` + +## ICE Restart Required Scenarios + +These events require ICE restart (see [patterns.md](./patterns.md#ice-restart-pattern)): + +1. **TURN server maintenance** (occasional on Cloudflare's network) +2. **Network topology changes** (anycast routing changes) +3. **Credential refresh** during long sessions (>1 hour) +4. **Connection failure** (iceConnectionState === 'failed') + +Implement in all production apps: + +```typescript +pc.addEventListener('iceconnectionstatechange', async () => { + if (pc.iceConnectionState === 'failed' || + pc.iceConnectionState === 'disconnected') { + await refreshTURNCredentials(pc); + pc.restartIce(); + const offer = await pc.createOffer({ iceRestart: true }); + await pc.setLocalDescription(offer); + // Send offer to peer via signaling... + } +}); +``` + +Reference: [RFC 8445 Section 2.4](https://datatracker.ietf.org/doc/html/rfc8445#section-2.4) + +## Security Checklist + +- [ ] Credentials generated server-side only (never client-side) +- [ ] TURN_KEY_SECRET in wrangler secrets, not vars +- [ ] TTL ≤ expected session duration (and ≤ 48 hours) +- [ ] Rate limiting on credential generation endpoint +- [ ] Client authentication before issuing credentials +- [ ] Credential revocation API for compromised sessions +- [ ] No hardcoded IPs (or DNS monitoring in place) +- [ ] Port 53 filtered for browser clients + +## Troubleshooting + +### Issue: TURN credentials not working + +**Check:** +- Key ID and secret are correct +- Credentials haven't expired (check TTL) +- TTL doesn't exceed 172800 seconds (48 hours) +- Server can reach rtc.live.cloudflare.com +- Network allows outbound HTTPS + +**Solution:** +```typescript +// Validate before using +if (ttl > 172800) { + throw new Error('TTL cannot exceed 48 hours'); +} +``` + +### Issue: Slow connection establishment + +**Solutions:** +- Ensure proper ICE candidate gathering +- Check network latency to Cloudflare edge +- Verify firewall allows WebRTC ports (3478, 5349, 443) +- Consider using TURN over TLS (port 443) for corporate networks + +### Issue: High packet loss + +**Check:** +- Not exceeding rate limits (5-10k pps) +- Not exceeding bandwidth limits (50-100 Mbps) +- Not connecting to too many unique IPs (>5/sec) +- Client network quality + +### Issue: Connection drops after ~48 hours + +**Cause**: Credentials expired (48hr max) + +**Solution**: +- Set TTL to expected session duration +- Implement credential refresh with setConfiguration() +- Use ICE restart if connection fails + +```typescript +// Refresh credentials before expiry +const refreshInterval = ttl * 1000 - 60000; // 1 min early +setInterval(async () => { + await refreshTURNCredentials(pc); +}, refreshInterval); +``` + +### Issue: Port 53 URLs in browser fail silently + +**Cause**: Chrome/Firefox block port 53 + +**Solution**: Filter port 53 URLs server-side: + +```typescript +const filtered = urls.filter(url => !url.includes(':53')); +``` + +### Issue: Hardcoded IPs stop working + +**Cause**: Cloudflare changed IP addresses (14-day notice) + +**Solution**: +- Use DNS hostnames (`turn.cloudflare.com`) +- Monitor DNS changes with automated alerts +- Update allowlists within 14 days if using IP allowlisting + +## Cost Optimization + +1. Use appropriate TTLs (don't over-provision) +2. Implement credential caching +3. Set `iceTransportPolicy: 'all'` to try direct first (use `'relay'` only when necessary) +4. Monitor bandwidth usage +5. Free when used with Cloudflare Calls SFU + +## See Also + +- [api.md](./api.md) - Credential generation API, revocation +- [configuration.md](./configuration.md) - IP allowlisting, monitoring +- [patterns.md](./patterns.md) - ICE restart, credential refresh patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/turn/patterns.md b/plugins/cloudflare/skills/cloudflare/references/turn/patterns.md new file mode 100644 index 0000000..39333be --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turn/patterns.md @@ -0,0 +1,213 @@ +# TURN Implementation Patterns + +Production-ready patterns for implementing Cloudflare TURN in WebRTC applications. + +## Prerequisites + +Before implementing these patterns, ensure you have: +- TURN key created: see [api.md#create-turn-key](./api.md#create-turn-key) +- Worker configured: see [configuration.md#cloudflare-worker-integration](./configuration.md#cloudflare-worker-integration) + +## Basic TURN Configuration (Browser) + +```typescript +interface RTCIceServer { + urls: string | string[]; + username?: string; + credential?: string; + credentialType?: "password" | "oauth"; +} + +async function getTURNConfig(): Promise { + const response = await fetch('/api/turn-credentials'); + const data = await response.json(); + + return [ + { + urls: 'stun:stun.cloudflare.com:3478' + }, + { + urls: [ + 'turn:turn.cloudflare.com:3478?transport=udp', + 'turn:turn.cloudflare.com:3478?transport=tcp', + 'turns:turn.cloudflare.com:5349?transport=tcp', + 'turns:turn.cloudflare.com:443?transport=tcp' + ], + username: data.username, + credential: data.credential, + credentialType: 'password' + } + ]; +} + +// Use in RTCPeerConnection +const iceServers = await getTURNConfig(); +const peerConnection = new RTCPeerConnection({ iceServers }); +``` + +## Port Selection Strategy + +Recommended order for browser clients: + +1. **3478/udp** (primary, lowest latency) +2. **3478/tcp** (fallback for UDP-blocked networks) +3. **5349/tls** (corporate firewalls, most reliable) +4. **443/tls** (alternate TLS port, firewall-friendly) + +**Avoid port 53**—blocked by Chrome and Firefox. + +```typescript +function filterICEServersForBrowser(urls: string[]): string[] { + return urls + .filter(url => !url.includes(':53')) // Remove port 53 + .sort((a, b) => { + // Prioritize UDP over TCP over TLS + if (a.includes('transport=udp')) return -1; + if (b.includes('transport=udp')) return 1; + if (a.includes('transport=tcp') && !a.startsWith('turns:')) return -1; + if (b.includes('transport=tcp') && !b.startsWith('turns:')) return 1; + return 0; + }); +} +``` + +## Credential Refresh (Mid-Session) + +When credentials expire during long calls: + +```typescript +async function refreshTURNCredentials(pc: RTCPeerConnection): Promise { + const newCreds = await fetch('/turn-credentials').then(r => r.json()); + const config = pc.getConfiguration(); + config.iceServers = newCreds.iceServers; + pc.setConfiguration(config); + // Note: setConfiguration() does NOT trigger ICE restart + // Combine with restartIce() if connection fails +} + +// Auto-refresh before expiry +setInterval(async () => { + await refreshTURNCredentials(peerConnection); +}, 3000000); // 50 minutes if TTL is 1 hour +``` + +## ICE Restart Pattern + +After network change, TURN server maintenance, or credential expiry: + +```typescript +pc.addEventListener('iceconnectionstatechange', async () => { + if (pc.iceConnectionState === 'failed') { + console.warn('ICE connection failed, restarting...'); + + // Refresh credentials + await refreshTURNCredentials(pc); + + // Trigger ICE restart + pc.restartIce(); + const offer = await pc.createOffer({ iceRestart: true }); + await pc.setLocalDescription(offer); + + // Send offer to peer via signaling channel... + } +}); +``` + +## Credentials Caching Pattern + +```typescript +class TURNCredentialsManager { + private creds: { username: string; credential: string; urls: string[]; expiresAt: number; } | null = null; + + async getCredentials(keyId: string, keySecret: string): Promise { + const now = Date.now(); + + if (this.creds && this.creds.expiresAt > now) { + return this.buildIceServers(this.creds); + } + + const ttl = 3600; + if (ttl > 172800) throw new Error('TTL max 48hrs'); + + const res = await fetch( + `https://rtc.live.cloudflare.com/v1/turn/keys/${keyId}/credentials/generate`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${keySecret}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ ttl }) + } + ); + + const data = await res.json(); + const filteredUrls = data.iceServers.urls.filter((url: string) => !url.includes(':53')); + + this.creds = { + username: data.iceServers.username, + credential: data.iceServers.credential, + urls: filteredUrls, + expiresAt: now + (ttl * 1000) - 60000 + }; + + return this.buildIceServers(this.creds); + } + + private buildIceServers(c: { username: string; credential: string; urls: string[] }): RTCIceServer[] { + return [ + { urls: 'stun:stun.cloudflare.com:3478' }, + { urls: c.urls, username: c.username, credential: c.credential, credentialType: 'password' as const } + ]; + } +} +``` + +## Common Use Cases + +```typescript +// Video conferencing: TURN as fallback +const config = { iceServers: await getTURNConfig(), iceTransportPolicy: 'all' }; + +// IoT/predictable connectivity: force TURN +const config = { iceServers: await getTURNConfig(), iceTransportPolicy: 'relay' }; + +// Screen sharing: reduce overhead +const pc = new RTCPeerConnection({ iceServers: await getTURNConfig(), bundlePolicy: 'max-bundle' }); +``` + +## Integration with Cloudflare Calls SFU + +```typescript +// TURN is automatically used when needed +// Cloudflare Calls handles TURN + SFU coordination +const session = await callsClient.createSession({ + appId: 'your-app-id', + sessionId: 'meeting-123' +}); +``` + +## Debugging ICE Connectivity + +```typescript +pc.addEventListener('icecandidate', (event) => { + if (event.candidate) { + console.log('ICE candidate:', event.candidate.type, event.candidate.protocol); + } +}); + +pc.addEventListener('iceconnectionstatechange', () => { + console.log('ICE state:', pc.iceConnectionState); +}); + +// Check selected candidate pair +const stats = await pc.getStats(); +stats.forEach(report => { + if (report.type === 'candidate-pair' && report.selected) { + console.log('Selected:', report); + } +}); +``` + +## See Also + +- [api.md](./api.md) - Credential generation API, types +- [configuration.md](./configuration.md) - Worker setup, environment variables +- [gotchas.md](./gotchas.md) - Common mistakes, troubleshooting diff --git a/plugins/cloudflare/skills/cloudflare/references/turnstile/README.md b/plugins/cloudflare/skills/cloudflare/references/turnstile/README.md new file mode 100644 index 0000000..29de799 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turnstile/README.md @@ -0,0 +1,99 @@ +# Cloudflare Turnstile Implementation Skill Reference + +Expert guidance for implementing Cloudflare Turnstile - a smart CAPTCHA alternative that protects websites from bots without showing traditional CAPTCHA puzzles. + +## Overview + +Turnstile is a user-friendly CAPTCHA alternative that runs challenges in the background without user interaction. It validates visitors automatically using signals like browser behavior, device fingerprinting, and machine learning. + +## Widget Types + +| Type | Interaction | Use Case | +|------|-------------|----------| +| **Managed** (default) | Shows checkbox when needed | Forms, logins - balance UX and security | +| **Non-Interactive** | Invisible, runs automatically | Frictionless UX, low-risk actions | +| **Invisible** | Hidden, triggered programmatically | Pre-clearance, API calls, headless | + +## Quick Start + +### Implicit Rendering (HTML-based) +```html + + + + +
+
+ +
+``` + +### Explicit Rendering (JavaScript-based) +```html +
+ + +``` + +### Server Validation (Required) +```javascript +// Cloudflare Workers +export default { + async fetch(request) { + const formData = await request.formData(); + const token = formData.get('cf-turnstile-response'); + + const result = await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + secret: env.TURNSTILE_SECRET, + response: token, + remoteip: request.headers.get('CF-Connecting-IP') + }) + }); + + const validation = await result.json(); + if (!validation.success) { + return new Response('Invalid CAPTCHA', { status: 400 }); + } + // Process form... + } +} +``` + +## Testing Keys + +**Critical for development/testing:** + +| Type | Key | Behavior | +|------|-----|----------| +| **Site Key (Always Passes)** | `1x00000000000000000000AA` | Widget succeeds, token validates | +| **Site Key (Always Blocks)** | `2x00000000000000000000AB` | Widget fails visibly | +| **Site Key (Force Challenge)** | `3x00000000000000000000FF` | Always shows interactive challenge | +| **Secret Key (Testing)** | `1x0000000000000000000000000000000AA` | Validates test tokens | + +**Note:** Test keys work on `localhost` and any domain. Do NOT use in production. + +## Key Constraints + +- **Token expiry:** 5 minutes after generation +- **Single-use:** Each token can only be validated once +- **Server validation required:** Client-side checks are insufficient + +## Reading Order + +1. **[configuration.md](configuration.md)** - Setup, widget options, script loading +2. **[api.md](api.md)** - JavaScript API, siteverify endpoints, TypeScript types +3. **[patterns.md](patterns.md)** - Form integration, framework examples, validation patterns +4. **[gotchas.md](gotchas.md)** - Common errors, debugging, limitations + +## See Also + +- [Cloudflare Turnstile Docs](https://developers.cloudflare.com/turnstile/) +- [Dashboard](https://dash.cloudflare.com/?to=/:account/turnstile) diff --git a/plugins/cloudflare/skills/cloudflare/references/turnstile/api.md b/plugins/cloudflare/skills/cloudflare/references/turnstile/api.md new file mode 100644 index 0000000..f65ee4f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turnstile/api.md @@ -0,0 +1,240 @@ +# API Reference + +## Client-Side JavaScript API + +The Turnstile JavaScript API is available at `window.turnstile` after loading the script. + +### `turnstile.render(container, options)` + +Renders a Turnstile widget into a container element. + +**Parameters:** +- `container` (string | HTMLElement): CSS selector or DOM element +- `options` (TurnstileOptions): Configuration object (see [configuration.md](configuration.md)) + +**Returns:** `string` - Widget ID for use with other API methods + +**Example:** +```javascript +const widgetId = window.turnstile.render('#my-container', { + sitekey: 'YOUR_SITE_KEY', + callback: (token) => console.log('Success:', token), + 'error-callback': (code) => console.error('Error:', code) +}); +``` + +### `turnstile.reset(widgetId)` + +Resets a widget (clears token, resets challenge state). Useful when form validation fails. + +**Parameters:** +- `widgetId` (string): Widget ID from `render()`, or container element + +**Returns:** `void` + +**Example:** +```javascript +// Reset on form error +if (!validateForm()) { + window.turnstile.reset(widgetId); +} +``` + +### `turnstile.remove(widgetId)` + +Removes a widget from the DOM completely. + +**Parameters:** +- `widgetId` (string): Widget ID from `render()` + +**Returns:** `void` + +**Example:** +```javascript +// Cleanup on navigation +window.turnstile.remove(widgetId); +``` + +### `turnstile.getResponse(widgetId)` + +Gets the current token from a widget (if challenge completed). + +**Parameters:** +- `widgetId` (string): Widget ID from `render()`, or container element + +**Returns:** `string | undefined` - Token string, or undefined if not ready + +**Example:** +```javascript +const token = window.turnstile.getResponse(widgetId); +if (token) { + submitForm(token); +} +``` + +### `turnstile.isExpired(widgetId)` + +Checks if a widget's token has expired (>5 minutes old). + +**Parameters:** +- `widgetId` (string): Widget ID from `render()` + +**Returns:** `boolean` - True if expired + +**Example:** +```javascript +if (window.turnstile.isExpired(widgetId)) { + window.turnstile.reset(widgetId); +} +``` + +## Callback Signatures + +```typescript +type TurnstileCallback = (token: string) => void; +type ErrorCallback = (errorCode: string) => void; +type TimeoutCallback = () => void; +type ExpiredCallback = () => void; +type BeforeInteractiveCallback = () => void; +type AfterInteractiveCallback = () => void; +type UnsupportedCallback = () => void; +``` + +## Siteverify API (Server-Side) + +**Endpoint:** `https://challenges.cloudflare.com/turnstile/v0/siteverify` + +### Request + +**Method:** POST +**Content-Type:** `application/json` or `application/x-www-form-urlencoded` + +```typescript +interface SiteverifyRequest { + secret: string; // Your secret key (never expose client-side) + response: string; // Token from cf-turnstile-response + remoteip?: string; // User's IP (optional but recommended) + idempotency_key?: string; // Unique key for idempotent validation +} +``` + +**Example:** +```javascript +// Cloudflare Workers +const result = await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + secret: env.TURNSTILE_SECRET, + response: token, + remoteip: request.headers.get('CF-Connecting-IP') + }) +}); +const data = await result.json(); +``` + +### Response + +```typescript +interface SiteverifyResponse { + success: boolean; // Validation result + challenge_ts?: string; // ISO timestamp of challenge + hostname?: string; // Hostname where widget was solved + 'error-codes'?: string[]; // Error codes if success=false + action?: string; // Action name from widget config + cdata?: string; // Custom data from widget config +} +``` + +**Example Success:** +```json +{ + "success": true, + "challenge_ts": "2024-01-15T10:30:00Z", + "hostname": "example.com", + "action": "login", + "cdata": "user123" +} +``` + +**Example Failure:** +```json +{ + "success": false, + "error-codes": ["timeout-or-duplicate"] +} +``` + +## Error Codes + +| Code | Cause | Solution | +|------|-------|----------| +| `missing-input-secret` | Secret key not provided | Include `secret` in request | +| `invalid-input-secret` | Secret key is wrong | Check secret key in dashboard | +| `missing-input-response` | Token not provided | Include `response` token | +| `invalid-input-response` | Token is invalid/malformed | Verify token from widget | +| `timeout-or-duplicate` | Token expired (>5min) or reused | Generate new token, validate once | +| `internal-error` | Cloudflare server error | Retry with exponential backoff | +| `bad-request` | Malformed request | Check JSON/form encoding | + +## TypeScript Types + +```typescript +interface TurnstileOptions { + sitekey: string; + action?: string; + cData?: string; + callback?: (token: string) => void; + 'error-callback'?: (errorCode: string) => void; + 'expired-callback'?: () => void; + 'timeout-callback'?: () => void; + 'before-interactive-callback'?: () => void; + 'after-interactive-callback'?: () => void; + 'unsupported-callback'?: () => void; + theme?: 'light' | 'dark' | 'auto'; + size?: 'normal' | 'compact' | 'flexible'; + tabindex?: number; + 'response-field'?: boolean; + 'response-field-name'?: string; + retry?: 'auto' | 'never'; + 'retry-interval'?: number; + language?: string; + execution?: 'render' | 'execute'; + appearance?: 'always' | 'execute' | 'interaction-only'; + 'refresh-expired'?: 'auto' | 'manual' | 'never'; +} + +interface Turnstile { + render(container: string | HTMLElement, options: TurnstileOptions): string; + reset(widgetId: string): void; + remove(widgetId: string): void; + getResponse(widgetId: string): string | undefined; + isExpired(widgetId: string): boolean; + execute(container?: string | HTMLElement, options?: TurnstileOptions): void; +} + +declare global { + interface Window { + turnstile: Turnstile; + onloadTurnstileCallback?: () => void; + } +} +``` + +## Script Loading + +```html + + + + + + + + + +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/turnstile/configuration.md b/plugins/cloudflare/skills/cloudflare/references/turnstile/configuration.md new file mode 100644 index 0000000..215bdab --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turnstile/configuration.md @@ -0,0 +1,222 @@ +# Configuration + +## Script Loading + +### Basic (Implicit Rendering) +```html + +``` +Automatically renders widgets with `class="cf-turnstile"` on page load. + +### Explicit Rendering +```html + +``` +Manual control over when/where widgets render via `window.turnstile.render()`. + +### With Load Callback +```html + + +``` + +### Compatibility Mode +```html + +``` +Provides `grecaptcha` API for Google reCAPTCHA drop-in replacement. + +## Widget Configuration + +### Complete Options Object + +```javascript +{ + // Required + sitekey: 'YOUR_SITE_KEY', // Widget sitekey from dashboard + + // Callbacks + callback: (token) => {}, // Success - token ready + 'error-callback': (code) => {}, // Error occurred + 'expired-callback': () => {}, // Token expired (>5min) + 'timeout-callback': () => {}, // Challenge timeout + 'before-interactive-callback': () => {}, // Before showing checkbox + 'after-interactive-callback': () => {}, // After user interacts + 'unsupported-callback': () => {}, // Browser doesn't support Turnstile + + // Appearance + theme: 'auto', // 'light' | 'dark' | 'auto' + size: 'normal', // 'normal' | 'compact' | 'flexible' + tabindex: 0, // Tab order (accessibility) + language: 'auto', // ISO 639-1 code or 'auto' + + // Behavior + execution: 'render', // 'render' (auto) | 'execute' (manual) + appearance: 'always', // 'always' | 'execute' | 'interaction-only' + retry: 'auto', // 'auto' | 'never' + 'retry-interval': 8000, // Retry interval (ms), default 8000 + 'refresh-expired': 'auto', // 'auto' | 'manual' | 'never' + + // Form Integration + 'response-field': true, // Add hidden input (default: true) + 'response-field-name': 'cf-turnstile-response', // Hidden input name + + // Analytics & Data + action: 'login', // Action name (for analytics) + cData: 'user-session-123', // Custom data (returned in siteverify) +} +``` + +### Key Options Explained + +**`execution`:** +- `'render'` (default): Challenge starts immediately on render +- `'execute'`: Wait for `turnstile.execute()` call + +**`appearance`:** +- `'always'` (default): Widget always visible +- `'execute'`: Hidden until `execute()` called +- `'interaction-only'`: Hidden until user interaction needed + +**`refresh-expired`:** +- `'auto'` (default): Auto-refresh expired tokens +- `'manual'`: App must call `reset()` after expiry +- `'never'`: No refresh, expired-callback triggered + +**`retry`:** +- `'auto'` (default): Auto-retry failed challenges +- `'never'`: Don't retry, trigger error-callback + +## HTML Data Attributes + +For implicit rendering, use data attributes on `
`: + +| JavaScript Property | HTML Data Attribute | Example | +|---------------------|---------------------|---------| +| `sitekey` | `data-sitekey` | `data-sitekey="YOUR_KEY"` | +| `action` | `data-action` | `data-action="login"` | +| `cData` | `data-cdata` | `data-cdata="session-123"` | +| `callback` | `data-callback` | `data-callback="onSuccess"` | +| `error-callback` | `data-error-callback` | `data-error-callback="onError"` | +| `expired-callback` | `data-expired-callback` | `data-expired-callback="onExpired"` | +| `timeout-callback` | `data-timeout-callback` | `data-timeout-callback="onTimeout"` | +| `theme` | `data-theme` | `data-theme="dark"` | +| `size` | `data-size` | `data-size="compact"` | +| `tabindex` | `data-tabindex` | `data-tabindex="0"` | +| `response-field` | `data-response-field` | `data-response-field="false"` | +| `response-field-name` | `data-response-field-name` | `data-response-field-name="token"` | +| `retry` | `data-retry` | `data-retry="never"` | +| `retry-interval` | `data-retry-interval` | `data-retry-interval="5000"` | +| `language` | `data-language` | `data-language="en"` | +| `execution` | `data-execution` | `data-execution="execute"` | +| `appearance` | `data-appearance` | `data-appearance="interaction-only"` | +| `refresh-expired` | `data-refresh-expired` | `data-refresh-expired="manual"` | + +**Example:** +```html +
+``` + +## Content Security Policy + +Add these directives to CSP header/meta tag: + +``` +script-src https://challenges.cloudflare.com; +frame-src https://challenges.cloudflare.com; +``` + +**Full Example:** +```html + +``` + +## Framework-Specific Setup + +### React +```bash +npm install @marsidev/react-turnstile +``` +```jsx +import Turnstile from '@marsidev/react-turnstile'; + + console.log(token)} +/> +``` + +### Vue +```bash +npm install vue-turnstile +``` +```vue + + +``` + +### Svelte +```bash +npm install svelte-turnstile +``` +```svelte + + +``` + +### Next.js (App Router) +```tsx +// app/components/TurnstileWidget.tsx +'use client'; +import { useEffect, useRef } from 'react'; + +export default function TurnstileWidget({ sitekey, onSuccess }) { + const ref = useRef(null); + + useEffect(() => { + if (ref.current && window.turnstile) { + const widgetId = window.turnstile.render(ref.current, { + sitekey, + callback: onSuccess + }); + return () => window.turnstile.remove(widgetId); + } + }, [sitekey, onSuccess]); + + return
; +} +``` + +## Cloudflare Pages Plugin + +```bash +npm install @cloudflare/pages-plugin-turnstile +``` + +```typescript +// functions/_middleware.ts +import turnstilePlugin from '@cloudflare/pages-plugin-turnstile'; + +export const onRequest = turnstilePlugin({ + secret: 'YOUR_SECRET_KEY', + onError: () => new Response('CAPTCHA failed', { status: 403 }) +}); +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/turnstile/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/turnstile/gotchas.md new file mode 100644 index 0000000..f7556b4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turnstile/gotchas.md @@ -0,0 +1,218 @@ +# Troubleshooting & Gotchas + +## Critical Rules + +### ❌ Skipping Server-Side Validation +**Problem:** Client-only validation is easily bypassed. + +**Solution:** Always validate on server. +```javascript +// CORRECT - Server validates token +app.post('/submit', async (req, res) => { + const token = req.body['cf-turnstile-response']; + const validation = await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify', { + method: 'POST', + body: JSON.stringify({ secret: SECRET, response: token }) + }).then(r => r.json()); + + if (!validation.success) return res.status(403).json({ error: 'CAPTCHA failed' }); +}); +``` + +### ❌ Exposing Secret Key +**Problem:** Secret key leaked in client-side code. + +**Solution:** Server-side validation only. Never send secret to client. + +### ❌ Reusing Tokens (Single-Use Rule) +**Problem:** Tokens are single-use. Revalidation fails with `timeout-or-duplicate`. + +**Solution:** Generate new token for each submission. Reset widget on error. +```javascript +if (!response.ok) window.turnstile.reset(widgetId); +``` + +### ❌ Not Handling Token Expiry +**Problem:** Tokens expire after 5 minutes. + +**Solution:** Handle expiry callback or use auto-refresh. +```javascript +window.turnstile.render('#container', { + sitekey: 'YOUR_SITE_KEY', + 'refresh-expired': 'auto', // or 'manual' with expired-callback + 'expired-callback': () => window.turnstile.reset(widgetId) +}); +``` + +## Common Errors + +| Error | Cause | Solution | +|-------|-------|----------| +| **Widget not rendering** | Incorrect sitekey, CSP blocking, file:// protocol | Check sitekey, add CSP for challenges.cloudflare.com, use http:// | +| **timeout-or-duplicate** | Token expired (>5min) or reused | Generate fresh token, don't cache >5min | +| **invalid-input-secret** | Wrong secret key | Verify secret from dashboard, check env vars | +| **missing-input-response** | Token not sent | Check form field name is 'cf-turnstile-response' | + +## Framework Gotchas + +### React: Widget Re-mounting +**Problem:** Widget re-renders on state change, losing token. + +**Solution:** Control lifecycle with useRef. +```tsx +function TurnstileWidget({ onToken }) { + const containerRef = useRef(null); + const widgetIdRef = useRef(null); + + useEffect(() => { + if (containerRef.current && !widgetIdRef.current) { + widgetIdRef.current = window.turnstile.render(containerRef.current, { + sitekey: 'YOUR_SITE_KEY', + callback: onToken + }); + } + return () => { + if (widgetIdRef.current) { + window.turnstile.remove(widgetIdRef.current); + widgetIdRef.current = null; + } + }; + }, []); + + return
; +} +``` + +### React StrictMode: Double Render +**Problem:** Widget renders twice in dev due to StrictMode. + +**Solution:** Use cleanup function. +```tsx +useEffect(() => { + const widgetId = window.turnstile.render('#container', { sitekey }); + return () => window.turnstile.remove(widgetId); +}, []); +``` + +### Next.js: SSR Hydration +**Problem:** `window.turnstile` undefined during SSR. + +**Solution:** Use `'use client'` or dynamic import with `ssr: false`. +```tsx +'use client'; +export default function Turnstile() { /* component */ } +``` + +### SPA: Navigation Without Cleanup +**Problem:** Navigating leaves orphaned widgets. + +**Solution:** Remove widget in cleanup. +```javascript +// Vue +onBeforeUnmount(() => window.turnstile.remove(widgetId)); + +// React +useEffect(() => () => window.turnstile.remove(widgetId), []); +``` + +## Network & Security + +### CSP Blocking +**Problem:** Content Security Policy blocks script/iframe. + +**Solution:** Add CSP directives. +```html + +``` + +### IP Address Forwarding +**Problem:** Server receives proxy IP instead of client IP. + +**Solution:** Use correct header. +```javascript +// Cloudflare Workers +const ip = request.headers.get('CF-Connecting-IP'); + +// Behind proxy +const ip = request.headers.get('X-Forwarded-For')?.split(',')[0]; +``` + +### CORS (Siteverify) +**Problem:** CORS error calling siteverify from browser. + +**Solution:** Never call siteverify client-side. Call your backend, backend calls siteverify. + +## Limits & Constraints + +| Limit | Value | Impact | +|-------|-------|--------| +| Token validity | 5 minutes | Must regenerate after expiry | +| Token use | Single-use | Cannot revalidate same token | +| Widget size | 300x65px (normal), 130x120px (compact) | Plan layout | + +## Debugging + +### Console Logging +```javascript +window.turnstile.render('#container', { + sitekey: 'YOUR_SITE_KEY', + callback: (token) => console.log('✓ Token:', token), + 'error-callback': (code) => console.error('✗ Error:', code), + 'expired-callback': () => console.warn('⏱ Expired'), + 'timeout-callback': () => console.warn('⏱ Timeout') +}); +``` + +### Check Token State +```javascript +const token = window.turnstile.getResponse(widgetId); +console.log('Token:', token || 'NOT READY'); +console.log('Expired:', window.turnstile.isExpired(widgetId)); +``` + +### Test Keys (Use First) +Always develop with test keys before production: +- Site: `1x00000000000000000000AA` +- Secret: `1x0000000000000000000000000000000AA` + +### Network Tab +- Verify `api.js` loads (200 OK) +- Check siteverify request/response +- Look for 4xx/5xx errors + +## Misconfigurations + +### Wrong Key Pairing +**Problem:** Site key from one widget, secret from another. + +**Solution:** Verify site key and secret are from same widget in dashboard. + +### Test Keys in Production +**Problem:** Using test keys in production. + +**Solution:** Environment-based keys. +```javascript +const SITE_KEY = process.env.NODE_ENV === 'production' + ? process.env.TURNSTILE_SITE_KEY + : '1x00000000000000000000AA'; +``` + +### Missing Environment Variables +**Problem:** Secret undefined on server. + +**Solution:** Check .env and verify loading. +```bash +# .env +TURNSTILE_SECRET=your_secret_here + +# Verify +console.log('Secret loaded:', !!process.env.TURNSTILE_SECRET); +``` + +## Reference + +- [Turnstile Docs](https://developers.cloudflare.com/turnstile/) +- [Dashboard](https://dash.cloudflare.com/?to=/:account/turnstile) +- [Error Codes](https://developers.cloudflare.com/turnstile/troubleshooting/) diff --git a/plugins/cloudflare/skills/cloudflare/references/turnstile/patterns.md b/plugins/cloudflare/skills/cloudflare/references/turnstile/patterns.md new file mode 100644 index 0000000..c147a6f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/turnstile/patterns.md @@ -0,0 +1,193 @@ +# Common Patterns + +## Form Integration + +### Basic Form (Implicit Rendering) + +```html + + + + + + +
+ +
+ +
+ + +``` + +### Controlled Form (Explicit Rendering) + +```javascript + + +``` + +## Framework Patterns + +### React + +```tsx +import { useState } from 'react'; +import Turnstile from '@marsidev/react-turnstile'; + +export default function Form() { + const [token, setToken] = useState(null); + + return ( +
{ + e.preventDefault(); + if (!token) return; + await fetch('/api/submit', { + method: 'POST', + body: JSON.stringify({ 'cf-turnstile-response': token }) + }); + }}> + + + + ); +} +``` + +### Vue / Svelte + +```vue + + + + + token = e.detail.token} /> +``` + +## Server Validation + +### Cloudflare Workers + +```typescript +interface Env { + TURNSTILE_SECRET: string; +} + +export default { + async fetch(request: Request, env: Env): Promise { + if (request.method !== 'POST') { + return new Response('Method not allowed', { status: 405 }); + } + + const formData = await request.formData(); + const token = formData.get('cf-turnstile-response'); + + if (!token) { + return new Response('Missing token', { status: 400 }); + } + + // Validate token + const ip = request.headers.get('CF-Connecting-IP'); + const result = await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + secret: env.TURNSTILE_SECRET, + response: token, + remoteip: ip + }) + }); + + const validation = await result.json(); + + if (!validation.success) { + return new Response('CAPTCHA validation failed', { status: 403 }); + } + + // Process form... + return new Response('Success'); + } +}; +``` + +### Pages Functions + +```typescript +// functions/submit.ts - same pattern as Workers, use ctx.env and ctx.request +export const onRequestPost: PagesFunction<{ TURNSTILE_SECRET: string }> = async (ctx) => { + const token = (await ctx.request.formData()).get('cf-turnstile-response'); + // Validate with ctx.env.TURNSTILE_SECRET (same as Workers pattern above) +}; +``` + +## Advanced Patterns + +### Pre-Clearance (Invisible) + +```html +
+ + + + +``` + +### Token Refresh on Expiry + +```javascript +let widgetId = window.turnstile.render('#container', { + sitekey: 'YOUR_SITE_KEY', + 'refresh-expired': 'manual', + 'expired-callback': () => { + console.log('Token expired, refreshing...'); + window.turnstile.reset(widgetId); + } +}); +``` + +## Testing + +### Environment-Based Keys + +```javascript +const SITE_KEY = process.env.NODE_ENV === 'production' + ? 'YOUR_PRODUCTION_SITE_KEY' + : '1x00000000000000000000AA'; // Always passes + +const SECRET_KEY = process.env.NODE_ENV === 'production' + ? process.env.TURNSTILE_SECRET + : '1x0000000000000000000000000000000AA'; +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/vectorize/README.md b/plugins/cloudflare/skills/cloudflare/references/vectorize/README.md new file mode 100644 index 0000000..9707a10 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/vectorize/README.md @@ -0,0 +1,133 @@ +# Cloudflare Vectorize + +Globally distributed vector database for AI applications. Store and query vector embeddings for semantic search, recommendations, RAG, and classification. + +**Status:** Generally Available (GA) | **Last Updated:** 2026-01-27 + +## Quick Start + +```typescript +// 1. Create index +// npx wrangler vectorize create my-index --dimensions=768 --metric=cosine + +// 2. Configure binding (wrangler.jsonc) +// { "vectorize": [{ "binding": "VECTORIZE", "index_name": "my-index" }] } + +// 3. Query vectors +const matches = await env.VECTORIZE.query(queryVector, { topK: 5 }); +``` + +## Key Features + +- **10M vectors per index** (V2) +- Dimensions up to 1536 (32-bit float) +- Three distance metrics: cosine, euclidean, dot-product +- Metadata filtering (up to 10 indexes) +- Namespace support (50K namespaces paid, 1K free) +- Seamless Workers AI integration +- Global distribution + +## Reading Order + +| Task | Files to Read | +|------|---------------| +| New to Vectorize | README only | +| Implement feature | README + api + patterns | +| Setup/configure | README + configuration | +| Debug issues | gotchas | +| Integrate with AI | README + patterns | +| RAG implementation | README + patterns | + +## File Guide + +- **README.md** (this file): Overview, quick decisions +- **api.md**: Runtime API, types, operations (query/insert/upsert) +- **configuration.md**: Setup, CLI, metadata indexes +- **patterns.md**: RAG, Workers AI, OpenAI, LangChain, multi-tenant +- **gotchas.md**: Limits, pitfalls, troubleshooting + +## Distance Metric Selection + +Choose based on your use case: + +``` +What are you building? +├─ Text/semantic search → cosine (most common) +├─ Image similarity → euclidean +├─ Recommendation system → dot-product +└─ Pre-normalized vectors → dot-product +``` + +| Metric | Best For | Score Interpretation | +|--------|----------|---------------------| +| `cosine` | Text embeddings, semantic similarity | Higher = closer (1.0 = identical) | +| `euclidean` | Absolute distance, spatial data | Lower = closer (0.0 = identical) | +| `dot-product` | Recommendations, normalized vectors | Higher = closer | + +**Note:** Index configuration is immutable. Cannot change dimensions or metric after creation. + +## Multi-Tenancy Strategy + +``` +How many tenants? +├─ < 50K tenants → Use namespaces (recommended) +│ ├─ Fastest (filter before vector search) +│ └─ Strict isolation +├─ > 50K tenants → Use metadata filtering +│ ├─ Slower (post-filter after vector search) +│ └─ Requires metadata index +└─ Per-tenant indexes → Only if compliance mandated + └─ 50K index limit per account (paid plan) +``` + +## Common Workflows + +### Semantic Search + +```typescript +// 1. Generate embedding +const result = await env.AI.run("@cf/baai/bge-base-en-v1.5", { text: [query] }); + +// 2. Query Vectorize +const matches = await env.VECTORIZE.query(result.data[0], { + topK: 5, + returnMetadata: "indexed" +}); +``` + +### RAG Pattern + +```typescript +// 1. Generate query embedding +const embedding = await env.AI.run("@cf/baai/bge-base-en-v1.5", { text: [query] }); + +// 2. Search Vectorize +const matches = await env.VECTORIZE.query(embedding.data[0], { topK: 5 }); + +// 3. Fetch full documents from R2/D1/KV +const docs = await Promise.all(matches.matches.map(m => + env.R2.get(m.metadata.key).then(obj => obj?.text()) +)); + +// 4. Generate LLM response with context +const answer = await env.AI.run("@cf/meta/llama-3-8b-instruct", { + prompt: `Context: ${docs.join("\n\n")}\n\nQuestion: ${query}\n\nAnswer:` +}); +``` + +## Critical Gotchas + +See `gotchas.md` for details. Most important: + +1. **Async mutations**: Inserts take 5-10s to be queryable +2. **500 batch limit**: Workers API enforces 500 vectors per call (undocumented) +3. **Metadata truncation**: `"indexed"` returns first 64 bytes only +4. **topK with metadata**: Max 20 (not 100) when using returnValues or returnMetadata: "all" +5. **Metadata indexes first**: Must create before inserting vectors + +## Resources + +- [Official Docs](https://developers.cloudflare.com/vectorize/) +- [Client API Reference](https://developers.cloudflare.com/vectorize/reference/client-api/) +- [Workers AI Models](https://developers.cloudflare.com/workers-ai/models/#text-embeddings) +- [Discord: #vectorize](https://discord.cloudflare.com) diff --git a/plugins/cloudflare/skills/cloudflare/references/vectorize/api.md b/plugins/cloudflare/skills/cloudflare/references/vectorize/api.md new file mode 100644 index 0000000..e29d87f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/vectorize/api.md @@ -0,0 +1,88 @@ +# Vectorize API Reference + +## Types + +```typescript +interface VectorizeVector { + id: string; // Max 64 bytes + values: number[]; // Must match index dimensions + namespace?: string; // Optional partition (max 64 bytes) + metadata?: Record; // Max 10 KiB +} +``` + +## Query + +```typescript +const matches = await env.VECTORIZE.query(queryVector, { + topK: 10, // Max 100 (or 20 with returnValues/returnMetadata:"all") + returnMetadata: "indexed", // "none" | "indexed" | "all" + returnValues: false, + namespace: "tenant-123", + filter: { category: "docs" } +}); +// matches.matches[0] = { id, score, metadata? } +``` + +**returnMetadata:** `"none"` (fastest) → `"indexed"` (recommended) → `"all"` (topK max 20) + +**queryById (V2 only):** Search using existing vector as query. +```typescript +await env.VECTORIZE.queryById("doc-123", { topK: 5 }); +``` + +## Insert/Upsert + +```typescript +// Insert: ignores duplicates (keeps first) +await env.VECTORIZE.insert([{ id, values, metadata }]); + +// Upsert: overwrites duplicates (keeps last) +await env.VECTORIZE.upsert([{ id, values, metadata }]); +``` + +**Max 500 vectors per call.** Queryable after 5-10 seconds. + +## Other Operations + +```typescript +// Get by IDs +const vectors = await env.VECTORIZE.getByIds(["id1", "id2"]); + +// Delete (max 1000 IDs per call) +await env.VECTORIZE.deleteByIds(["id1", "id2"]); + +// Index info +const info = await env.VECTORIZE.describe(); +// { dimensions, metric, vectorCount } +``` + +## Filtering + +Requires metadata index. Filter operators: + +| Operator | Example | +|----------|---------| +| `$eq` (implicit) | `{ category: "docs" }` | +| `$ne` | `{ status: { $ne: "deleted" } }` | +| `$in` / `$nin` | `{ tag: { $in: ["sale"] } }` | +| `$lt`, `$lte`, `$gt`, `$gte` | `{ price: { $lt: 100 } }` | + +**Constraints:** Max 2048 bytes, no dots/`$` in keys, values: string/number/boolean/null. + +## Performance + +| Configuration | topK Limit | Speed | +|--------------|------------|-------| +| No metadata | 100 | Fastest | +| `returnMetadata: "indexed"` | 100 | Fast | +| `returnMetadata: "all"` | 20 | Slower | +| `returnValues: true` | 20 | Slower | + +**Batch operations:** Always batch (500/call) for optimal throughput. + +```typescript +for (let i = 0; i < vectors.length; i += 500) { + await env.VECTORIZE.upsert(vectors.slice(i, i + 500)); +} +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/vectorize/configuration.md b/plugins/cloudflare/skills/cloudflare/references/vectorize/configuration.md new file mode 100644 index 0000000..8c64d3e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/vectorize/configuration.md @@ -0,0 +1,88 @@ +# Vectorize Configuration + +## Create Index + +```bash +npx wrangler vectorize create my-index --dimensions=768 --metric=cosine +``` + +**⚠️ Dimensions and metric are immutable** - cannot change after creation. + +## Worker Binding + +```jsonc +// wrangler.jsonc +{ + "vectorize": [ + { "binding": "VECTORIZE", "index_name": "my-index" } + ] +} +``` + +```typescript +interface Env { + VECTORIZE: Vectorize; +} +``` + +## Metadata Indexes + +**Must create BEFORE inserting vectors** - existing vectors not retroactively indexed. + +```bash +wrangler vectorize create-metadata-index my-index --property-name=category --type=string +wrangler vectorize create-metadata-index my-index --property-name=price --type=number +``` + +| Type | Use For | +|------|---------| +| `string` | Categories, tags (first 64 bytes indexed) | +| `number` | Prices, timestamps | +| `boolean` | Flags | + +## CLI Commands + +```bash +# Index management +wrangler vectorize list +wrangler vectorize info +wrangler vectorize delete + +# Vector operations +wrangler vectorize insert --file=embeddings.ndjson +wrangler vectorize get --ids=id1,id2 +wrangler vectorize delete-by-ids --ids=id1,id2 + +# Metadata indexes +wrangler vectorize list-metadata-index +wrangler vectorize delete-metadata-index --property-name=field +``` + +## Bulk Upload (NDJSON) + +```json +{"id": "1", "values": [0.1, 0.2, ...], "metadata": {"category": "docs"}} +{"id": "2", "values": [0.4, 0.5, ...], "namespace": "tenant-abc"} +``` + +**Limits:** 5000 vectors per file, 100 MB max + +## Cardinality Best Practice + +Bucket high-cardinality data: +```typescript +// ❌ Millisecond timestamps +metadata: { timestamp: Date.now() } + +// ✅ 5-minute buckets +metadata: { timestamp_bucket: Math.floor(Date.now() / 300000) * 300000 } +``` + +## Production Checklist + +1. Create index with correct dimensions +2. Create metadata indexes FIRST +3. Test bulk upload +4. Configure bindings +5. Deploy Worker +6. Verify queries diff --git a/plugins/cloudflare/skills/cloudflare/references/vectorize/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/vectorize/gotchas.md new file mode 100644 index 0000000..9282771 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/vectorize/gotchas.md @@ -0,0 +1,76 @@ +# Vectorize Gotchas + +## Critical Warnings + +### Async Mutations +Insert/upsert/delete return immediately but vectors aren't queryable for 5-10 seconds. + +### Batch Size Limit +**Workers API: 500 vectors max per call** (undocumented, silently truncates) + +```typescript +// ✅ Chunk into 500 +for (let i = 0; i < vectors.length; i += 500) { + await env.VECTORIZE.upsert(vectors.slice(i, i + 500)); +} +``` + +### Metadata Truncation +`returnMetadata: "indexed"` returns only first 64 bytes of strings. Use `"all"` for complete metadata (but max topK drops to 20). + +### topK Limits + +| returnMetadata | returnValues | Max topK | +|----------------|--------------|----------| +| `"none"` / `"indexed"` | `false` | 100 | +| `"all"` | any | **20** | +| any | `true` | **20** | + +### Metadata Indexes First +Create BEFORE inserting - existing vectors not retroactively indexed. + +```bash +# ✅ Create index FIRST +wrangler vectorize create-metadata-index my-index --property-name=category --type=string +wrangler vectorize insert my-index --file=data.ndjson +``` + +### Index Config Immutable +Cannot change dimensions/metric after creation. Must create new index and migrate. + +## Limits (V2) + +| Resource | Limit | +|----------|-------| +| Vectors per index | 10,000,000 | +| Max dimensions | 1536 | +| Batch upsert (Workers) | **500** | +| Indexed string metadata | **64 bytes** | +| Metadata indexes | 10 | +| Namespaces | 50,000 (paid) / 1,000 (free) | + +## Common Mistakes + +1. **Wrong embedding shape:** Extract `result.data[0]` from Workers AI +2. **Metadata index after data:** Re-upsert all vectors +3. **Insert vs upsert:** `insert` ignores duplicates, `upsert` overwrites +4. **Not batching:** Individual inserts ~1K/min, batched ~200K+/min + +## Troubleshooting + +**No results?** +- Wait 5-10s after insert +- Check namespace spelling (case-sensitive) +- Verify metadata index exists +- Check dimension mismatch + +**Metadata filter not working?** +- Index must exist before data insert +- Strings >64 bytes truncated +- Use dot notation for nested: `"product.category"` + +## Model Dimensions + +- `@cf/baai/bge-small-en-v1.5`: 384 +- `@cf/baai/bge-base-en-v1.5`: 768 +- `@cf/baai/bge-large-en-v1.5`: 1024 diff --git a/plugins/cloudflare/skills/cloudflare/references/vectorize/patterns.md b/plugins/cloudflare/skills/cloudflare/references/vectorize/patterns.md new file mode 100644 index 0000000..9ffa2cb --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/vectorize/patterns.md @@ -0,0 +1,90 @@ +# Vectorize Patterns + +## Workers AI Integration + +```typescript +// Generate embedding + query +const result = await env.AI.run("@cf/baai/bge-base-en-v1.5", { text: [query] }); +const matches = await env.VECTORIZE.query(result.data[0], { topK: 5 }); // Pass data[0]! +``` + +| Model | Dimensions | +|-------|------------| +| `@cf/baai/bge-small-en-v1.5` | 384 | +| `@cf/baai/bge-base-en-v1.5` | 768 (recommended) | +| `@cf/baai/bge-large-en-v1.5` | 1024 | + +## OpenAI Integration + +```typescript +const response = await openai.embeddings.create({ model: "text-embedding-ada-002", input: query }); +const matches = await env.VECTORIZE.query(response.data[0].embedding, { topK: 5 }); +``` + +## RAG Pattern + +```typescript +// 1. Embed query +const emb = await env.AI.run("@cf/baai/bge-base-en-v1.5", { text: [query] }); + +// 2. Search vectors +const matches = await env.VECTORIZE.query(emb.data[0], { topK: 5, returnMetadata: "indexed" }); + +// 3. Fetch full docs from R2/D1/KV +const docs = await Promise.all(matches.matches.map(m => env.R2.get(m.metadata.key).then(o => o?.text()))); + +// 4. Generate with context +const answer = await env.AI.run("@cf/meta/llama-3-8b-instruct", { + prompt: `Context:\n${docs.filter(Boolean).join("\n\n")}\n\nQuestion: ${query}\n\nAnswer:` +}); +``` + +## Multi-Tenant + +### Namespaces (< 50K tenants, fastest) + +```typescript +await env.VECTORIZE.upsert([{ id: "1", values: emb, namespace: `tenant-${id}` }]); +await env.VECTORIZE.query(vec, { namespace: `tenant-${id}`, topK: 10 }); +``` + +### Metadata Filter (> 50K tenants) + +```bash +wrangler vectorize create-metadata-index my-index --property-name=tenantId --type=string +``` + +```typescript +await env.VECTORIZE.upsert([{ id: "1", values: emb, metadata: { tenantId: id } }]); +await env.VECTORIZE.query(vec, { filter: { tenantId: id }, topK: 10 }); +``` + +## Hybrid Search + +```typescript +const matches = await env.VECTORIZE.query(vec, { + topK: 20, + filter: { + category: { $in: ["tech", "science"] }, + published: { $gte: lastMonthTimestamp } + } +}); +``` + +## Batch Ingestion + +```typescript +const BATCH = 500; +for (let i = 0; i < vectors.length; i += BATCH) { + await env.VECTORIZE.upsert(vectors.slice(i, i + BATCH)); +} +``` + +## Best Practices + +1. **Pass `data[0]`** not `data` or full response +2. **Batch 500** vectors per upsert +3. **Create metadata indexes** before inserting +4. **Use namespaces** for tenant isolation (faster than filters) +5. **`returnMetadata: "indexed"`** for best speed/data balance +6. **Handle 5-10s mutation delay** in async operations diff --git a/plugins/cloudflare/skills/cloudflare/references/waf/README.md b/plugins/cloudflare/skills/cloudflare/references/waf/README.md new file mode 100644 index 0000000..052d0ae --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/waf/README.md @@ -0,0 +1,113 @@ +# Cloudflare WAF Expert Skill Reference + +**Expertise**: Cloudflare Web Application Firewall (WAF) configuration, custom rules, managed rulesets, rate limiting, attack detection, and API integration + +## Overview + +Cloudflare WAF protects web applications from attacks through managed rulesets and custom rules. + +**Detection (Managed Rulesets)** +- Pre-configured rules maintained by Cloudflare +- CVE-based rules, OWASP Top 10 coverage +- Three main rulesets: Cloudflare Managed, OWASP CRS, Exposed Credentials +- Actions: log, block, challenge, js_challenge, managed_challenge + +**Mitigation (Custom Rules & Rate Limiting)** +- Custom expressions using Wirefilter syntax +- Attack score-based blocking (`cf.waf.score`) +- Rate limiting with per-IP, per-user, or custom characteristics +- Actions: block, challenge, js_challenge, managed_challenge, log, skip + +## Quick Start + +### Deploy Cloudflare Managed Ruleset +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: process.env.CF_API_TOKEN }); + +// Deploy managed ruleset to zone +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_request_firewall_managed', + name: 'Deploy Cloudflare Managed Ruleset', + rules: [{ + action: 'execute', + action_parameters: { + id: 'efb7b8c949ac4650a09736fc376e9aee', // Cloudflare Managed Ruleset + }, + expression: 'true', + enabled: true, + }], +}); +``` + +### Create Custom Rule +```typescript +// Block requests with attack score >= 40 +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_request_firewall_custom', + name: 'Custom WAF Rules', + rules: [{ + action: 'block', + expression: 'cf.waf.score gt 40', + description: 'Block high attack scores', + enabled: true, + }], +}); +``` + +### Create Rate Limit +```typescript +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_ratelimit', + name: 'API Rate Limits', + rules: [{ + action: 'block', + expression: 'http.request.uri.path eq "/api/login"', + action_parameters: { + ratelimit: { + characteristics: ['cf.colo.id', 'ip.src'], + period: 60, + requests_per_period: 10, + mitigation_timeout: 600, + }, + }, + enabled: true, + }], +}); +``` + +## Managed Ruleset Quick Reference + +| Ruleset Name | ID | Coverage | +|--------------|----|---------| +| Cloudflare Managed | `efb7b8c949ac4650a09736fc376e9aee` | OWASP Top 10, CVEs | +| OWASP Core Ruleset | `4814384a9e5d4991b9815dcfc25d2f1f` | OWASP ModSecurity CRS | +| Exposed Credentials Check | `c2e184081120413c86c3ab7e14069605` | Credential stuffing | + +## Phases + +WAF rules execute in specific phases: +- `http_request_firewall_managed` - Managed rulesets +- `http_request_firewall_custom` - Custom rules +- `http_ratelimit` - Rate limiting rules +- `http_request_sbfm` - Super Bot Fight Mode (Pro+) + +## Reading Order + +1. **[api.md](api.md)** - SDK methods, expressions, actions, parameters +2. **[configuration.md](configuration.md)** - Setup with Wrangler, Terraform, Pulumi +3. **[patterns.md](patterns.md)** - Common patterns: deploy managed, rate limiting, skip, override +4. **[gotchas.md](gotchas.md)** - Execution order, limits, expression errors + +## See Also + +- [Cloudflare WAF Docs](https://developers.cloudflare.com/waf/) +- [Ruleset Engine](https://developers.cloudflare.com/ruleset-engine/) +- [Expression Reference](https://developers.cloudflare.com/ruleset-engine/rules-language/) \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/waf/api.md b/plugins/cloudflare/skills/cloudflare/references/waf/api.md new file mode 100644 index 0000000..a7bc9e0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/waf/api.md @@ -0,0 +1,202 @@ +# API Reference + +## SDK Setup + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ + apiToken: process.env.CF_API_TOKEN, +}); +``` + +## Core Methods + +```typescript +// List rulesets +await client.rulesets.list({ zone_id: 'zone_id', phase: 'http_request_firewall_managed' }); + +// Get ruleset +await client.rulesets.get({ zone_id: 'zone_id', ruleset_id: 'ruleset_id' }); + +// Create ruleset +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_request_firewall_custom', + name: 'Custom WAF Rules', + rules: [{ action: 'block', expression: 'cf.waf.score gt 40', enabled: true }], +}); + +// Update ruleset (include rule id to keep existing, omit id for new rules) +await client.rulesets.update({ + zone_id: 'zone_id', + ruleset_id: 'ruleset_id', + rules: [ + { id: 'rule_id', action: 'block', expression: 'cf.waf.score gt 40', enabled: true }, + { action: 'challenge', expression: 'http.request.uri.path contains "/admin"', enabled: true }, + ], +}); + +// Delete ruleset +await client.rulesets.delete({ zone_id: 'zone_id', ruleset_id: 'ruleset_id' }); +``` + +## Actions & Phases + +### Actions by Phase + +| Action | Custom | Managed | Rate Limit | Description | +|--------|--------|---------|------------|-------------| +| `block` | ✅ | ❌ | ✅ | Block request with 403 | +| `challenge` | ✅ | ❌ | ✅ | Show CAPTCHA challenge | +| `js_challenge` | ✅ | ❌ | ✅ | JS-based challenge | +| `managed_challenge` | ✅ | ❌ | ✅ | Smart challenge (recommended) | +| `log` | ✅ | ❌ | ✅ | Log only, don't block | +| `skip` | ✅ | ❌ | ❌ | Skip rule evaluation | +| `execute` | ❌ | ✅ | ❌ | Deploy managed ruleset | + +### Phases (Execution Order) + +1. `http_request_firewall_custom` - Custom rules (first line of defense) +2. `http_request_firewall_managed` - Managed rulesets (pre-configured protection) +3. `http_ratelimit` - Rate limiting (request throttling) +4. `http_request_sbfm` - Super Bot Fight Mode (Pro+ only) + +## Expression Syntax + +### Fields + +```typescript +// Request properties +http.request.method // GET, POST, etc. +http.request.uri.path // /api/users +http.host // example.com + +// IP and Geolocation +ip.src // 192.0.2.1 +ip.geoip.country // US, GB, etc. +ip.geoip.continent // NA, EU, etc. + +// Attack detection +cf.waf.score // 0-100 attack score +cf.waf.score.sqli // SQL injection score +cf.waf.score.xss // XSS score + +// Headers & Cookies +http.request.headers["authorization"][0] +http.request.cookies["session"][0] +lower(http.user_agent) // Lowercase user agent +``` + +### Operators + +```typescript +// Comparison +eq // Equal +ne // Not equal +lt // Less than +le // Less than or equal +gt // Greater than +ge // Greater than or equal + +// String matching +contains // Substring match +matches // Regex match (use carefully) +starts_with // Prefix match +ends_with // Suffix match + +// List operations +in // Value in list +not // Logical NOT +and // Logical AND +or // Logical OR +``` + +### Expression Examples + +```typescript +'cf.waf.score gt 40' // Attack score +'http.request.uri.path eq "/api/login" and http.request.method eq "POST"' // Path + method +'ip.src in {192.0.2.0/24 203.0.113.0/24}' // IP blocking +'ip.geoip.country in {"CN" "RU" "KP"}' // Country blocking +'http.user_agent contains "bot"' // User agent +'not http.request.headers["authorization"][0]' // Header check +'(cf.waf.score.sqli gt 20 or cf.waf.score.xss gt 20) and http.request.uri.path starts_with "/api"' // Complex +``` + +## Rate Limiting Configuration + +```typescript +{ + action: 'block', + expression: 'http.request.uri.path starts_with "/api"', + action_parameters: { + ratelimit: { + // Characteristics define uniqueness: 'ip.src', 'cf.colo.id', + // 'http.request.headers["key"][0]', 'http.request.cookies["session"][0]' + characteristics: ['cf.colo.id', 'ip.src'], // Recommended: per-IP per-datacenter + period: 60, // Time window in seconds + requests_per_period: 100, // Max requests in period + mitigation_timeout: 600, // Block duration in seconds + counting_expression: 'http.request.method ne "GET"', // Optional: filter counted requests + requests_to_origin: false, // Count all requests (not just origin hits) + }, + }, + enabled: true, +} +``` + +## Managed Ruleset Deployment + +```typescript +{ + action: 'execute', + action_parameters: { + id: 'efb7b8c949ac4650a09736fc376e9aee', // Cloudflare Managed + overrides: { + // Override specific rules + rules: [ + { id: '5de7edfa648c4d6891dc3e7f84534ffa', action: 'log', enabled: true }, + ], + // Override categories: 'wordpress', 'sqli', 'xss', 'rce', etc. + categories: [ + { category: 'wordpress', enabled: false }, + { category: 'sqli', action: 'log' }, + ], + }, + }, + expression: 'true', + enabled: true, +} +``` + +## Skip Rules + +Skip rules bypass subsequent rule evaluation. Two skip types: + +**Skip current ruleset**: Skip remaining rules in current phase only +```typescript +{ + action: 'skip', + action_parameters: { + ruleset: 'current', // Skip rest of current ruleset + }, + expression: 'http.request.uri.path ends_with ".jpg" or http.request.uri.path ends_with ".css"', + enabled: true, +} +``` + +**Skip entire phases**: Skip one or more phases completely +```typescript +{ + action: 'skip', + action_parameters: { + phases: ['http_request_firewall_managed', 'http_ratelimit'], // Skip multiple phases + }, + expression: 'ip.src in {192.0.2.0/24 203.0.113.0/24}', + enabled: true, +} +``` + +**Note**: Skip rules in custom phase can skip managed/ratelimit phases, but not vice versa (execution order). \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/waf/configuration.md b/plugins/cloudflare/skills/cloudflare/references/waf/configuration.md new file mode 100644 index 0000000..796a291 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/waf/configuration.md @@ -0,0 +1,203 @@ +# Configuration + +## Prerequisites + +**API Token**: Create at https://dash.cloudflare.com/profile/api-tokens +- Permission: `Zone.WAF Edit` or `Zone.Firewall Services Edit` +- Zone Resources: Include specific zones or all zones + +**Zone ID**: Found in dashboard > Overview > API section (right sidebar) + +```bash +# Set environment variables +export CF_API_TOKEN="your_api_token_here" +export ZONE_ID="your_zone_id_here" +``` + +## TypeScript SDK Usage + +```bash +npm install cloudflare +``` + +```typescript +import Cloudflare from 'cloudflare'; + +const client = new Cloudflare({ apiToken: process.env.CF_API_TOKEN }); + +// Custom rules +await client.rulesets.create({ + zone_id: process.env.ZONE_ID, + kind: 'zone', + phase: 'http_request_firewall_custom', + name: 'Custom WAF', + rules: [ + { action: 'block', expression: 'cf.waf.score gt 50', enabled: true }, + { action: 'challenge', expression: 'http.request.uri.path eq "/admin"', enabled: true }, + ], +}); + +// Managed ruleset +await client.rulesets.create({ + zone_id: process.env.ZONE_ID, + phase: 'http_request_firewall_managed', + rules: [{ + action: 'execute', + action_parameters: { id: 'efb7b8c949ac4650a09736fc376e9aee' }, + expression: 'true', + }], +}); + +// Rate limiting +await client.rulesets.create({ + zone_id: process.env.ZONE_ID, + phase: 'http_ratelimit', + rules: [{ + action: 'block', + expression: 'http.request.uri.path starts_with "/api"', + action_parameters: { + ratelimit: { + characteristics: ['cf.colo.id', 'ip.src'], + period: 60, + requests_per_period: 100, + mitigation_timeout: 600, + }, + }, + }], +}); +``` + +## Terraform Configuration + +```hcl +provider "cloudflare" { + api_token = var.cloudflare_api_token +} + +resource "cloudflare_ruleset" "waf_custom" { + zone_id = var.zone_id + kind = "zone" + phase = "http_request_firewall_custom" + + rules { + action = "block" + expression = "cf.waf.score gt 50" + } +} +``` + +**Managed Ruleset & Rate Limiting**: +```hcl +resource "cloudflare_ruleset" "waf_managed" { + zone_id = var.zone_id + name = "Managed Ruleset" + kind = "zone" + phase = "http_request_firewall_managed" + + rules { + action = "execute" + action_parameters { + id = "efb7b8c949ac4650a09736fc376e9aee" + overrides { + rules { + id = "5de7edfa648c4d6891dc3e7f84534ffa" + action = "log" + } + } + } + expression = "true" + } +} + +resource "cloudflare_ruleset" "rate_limiting" { + zone_id = var.zone_id + phase = "http_ratelimit" + + rules { + action = "block" + expression = "http.request.uri.path starts_with \"/api\"" + ratelimit { + characteristics = ["cf.colo.id", "ip.src"] + period = 60 + requests_per_period = 100 + mitigation_timeout = 600 + } + } +} +``` + +## Pulumi Configuration + +```typescript +import * as cloudflare from '@pulumi/cloudflare'; + +const zoneId = 'zone_id'; + +// Custom rules +const wafCustom = new cloudflare.Ruleset('waf-custom', { + zoneId, + phase: 'http_request_firewall_custom', + rules: [ + { action: 'block', expression: 'cf.waf.score gt 50', enabled: true }, + { action: 'challenge', expression: 'http.request.uri.path eq "/admin"', enabled: true }, + ], +}); + +// Managed ruleset +const wafManaged = new cloudflare.Ruleset('waf-managed', { + zoneId, + phase: 'http_request_firewall_managed', + rules: [{ + action: 'execute', + actionParameters: { id: 'efb7b8c949ac4650a09736fc376e9aee' }, + expression: 'true', + }], +}); + +// Rate limiting +const rateLimiting = new cloudflare.Ruleset('rate-limiting', { + zoneId, + phase: 'http_ratelimit', + rules: [{ + action: 'block', + expression: 'http.request.uri.path starts_with "/api"', + ratelimit: { + characteristics: ['cf.colo.id', 'ip.src'], + period: 60, + requestsPerPeriod: 100, + mitigationTimeout: 600, + }, + }], +}); +``` + +## Dashboard Configuration + +1. Navigate to: **Security** > **WAF** +2. Select tab: + - **Managed rules** - Deploy/configure managed rulesets + - **Custom rules** - Create custom rules + - **Rate limiting rules** - Configure rate limits +3. Click **Deploy** or **Create rule** + +**Testing**: Use Security Events to test expressions before deploying. + +## Wrangler Integration + +WAF configuration is zone-level (not Worker-specific). Configuration methods: +- Dashboard UI +- Cloudflare API via SDK +- Terraform/Pulumi (IaC) + +**Workers benefit from WAF automatically** - no Worker code changes needed. + +**Example: Query WAF API from Worker**: +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + return fetch(`https://api.cloudflare.com/client/v4/zones/${env.ZONE_ID}/rulesets`, { + headers: { 'Authorization': `Bearer ${env.CF_API_TOKEN}` }, + }); + }, +}; +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/waf/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/waf/gotchas.md new file mode 100644 index 0000000..05b0dc1 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/waf/gotchas.md @@ -0,0 +1,204 @@ +# Gotchas & Troubleshooting + +## Execution Order + +**Problem:** Rules execute in unexpected order +**Cause:** Misunderstanding phase execution +**Solution:** + +Phases execute sequentially (can't be changed): +1. `http_request_firewall_custom` - Custom rules +2. `http_request_firewall_managed` - Managed rulesets +3. `http_ratelimit` - Rate limiting + +Within phase: top-to-bottom, first match wins (unless `skip`) + +```typescript +// WRONG: Can't mix phase-specific actions +await client.rulesets.create({ + phase: 'http_request_firewall_custom', + rules: [ + { action: 'block', expression: 'cf.waf.score gt 50' }, + { action: 'execute', action_parameters: { id: 'managed_id' } }, // WRONG + ], +}); + +// CORRECT: Separate rulesets per phase +await client.rulesets.create({ phase: 'http_request_firewall_custom', rules: [...] }); +await client.rulesets.create({ phase: 'http_request_firewall_managed', rules: [...] }); +``` + +## Expression Errors + +**Problem:** Syntax errors prevent deployment +**Cause:** Invalid field/operator/syntax +**Solution:** + +```typescript +// Common mistakes +'http.request.path' → 'http.request.uri.path' // Correct field +'ip.geoip.country eq US' → 'ip.geoip.country eq "US"' // Quote strings +'http.user_agent eq "Mozilla"' → 'lower(http.user_agent) contains "mozilla"' // Case sensitivity +'matches ".*[.jpg"' → 'matches ".*\\.jpg$"' // Valid regex +``` + +Test expressions in Security Events before deploying. + +## Skip Rule Pitfalls + +**Problem:** Skip rules don't work as expected +**Cause:** Misunderstanding skip scope +**Solution:** + +Skip types: +- `ruleset: 'current'` - Skip remaining rules in current ruleset only +- `phases: ['phase_name']` - Skip entire phases + +```typescript +// WRONG: Trying to skip managed rules from custom phase +// In http_request_firewall_custom: +{ + action: 'skip', + action_parameters: { ruleset: 'current' }, + expression: 'ip.src in {192.0.2.0/24}', +} +// This only skips remaining custom rules, not managed rules + +// CORRECT: Skip specific phases +{ + action: 'skip', + action_parameters: { + phases: ['http_request_firewall_managed', 'http_ratelimit'], + }, + expression: 'ip.src in {192.0.2.0/24}', +} +``` + +## Update Replaces All Rules + +**Problem:** Updating ruleset deletes other rules +**Cause:** `update()` replaces entire rule list +**Solution:** + +```typescript +// WRONG: This deletes all existing rules! +await client.rulesets.update({ + zone_id: 'zone_id', + ruleset_id: 'ruleset_id', + rules: [{ action: 'block', expression: 'cf.waf.score gt 50' }], +}); + +// CORRECT: Get existing rules first +const ruleset = await client.rulesets.get({ zone_id: 'zone_id', ruleset_id: 'ruleset_id' }); +await client.rulesets.update({ + zone_id: 'zone_id', + ruleset_id: 'ruleset_id', + rules: [...ruleset.rules, { action: 'block', expression: 'cf.waf.score gt 50' }], +}); +``` + +## Override Conflicts + +**Problem:** Managed ruleset overrides don't apply +**Cause:** Rule ID doesn't exist or category name incorrect +**Solution:** + +```typescript +// List managed ruleset rules to find IDs +const ruleset = await client.rulesets.get({ + zone_id: 'zone_id', + ruleset_id: 'efb7b8c949ac4650a09736fc376e9aee', +}); +console.log(ruleset.rules.map(r => ({ id: r.id, description: r.description }))); + +// Use correct IDs in overrides +{ action: 'execute', action_parameters: { id: 'efb7b8c949ac4650a09736fc376e9aee', + overrides: { rules: [{ id: '5de7edfa648c4d6891dc3e7f84534ffa', action: 'log' }] } } } +``` + +## False Positives + +**Problem:** Legitimate traffic blocked +**Cause:** Aggressive rules/thresholds +**Solution:** + +1. Start with log mode: `overrides: { action: 'log' }` +2. Review Security Events to identify false positives +3. Override specific rules: `overrides: { rules: [{ id: 'rule_id', action: 'log' }] }` + +## Rate Limiting NAT Issues + +**Problem:** Users behind NAT hit rate limits too quickly +**Cause:** Multiple users sharing single IP +**Solution:** + +Add more characteristics: User-Agent, session cookie, or authorization header +```typescript +{ + action: 'block', + expression: 'http.request.uri.path starts_with "/api"', + action_parameters: { + ratelimit: { + characteristics: ['cf.colo.id', 'ip.src', 'http.request.cookies["session"][0]'], + period: 60, + requests_per_period: 100, + }, + }, +} +``` + +## Performance Issues + +**Problem:** Increased latency +**Cause:** Complex expressions, excessive rules +**Solution:** + +1. Skip static assets early: `action: 'skip'` for `\\.(jpg|css|js)$` +2. Path-based deployment: Only run managed on `/api` or `/admin` +3. Disable unused categories: `{ category: 'wordpress', enabled: false }` +4. Prefer string operators over regex: `starts_with` vs `matches` + +## Limits & Quotas + +| Resource | Free | Pro | Business | Enterprise | +|----------|------|-----|----------|------------| +| Custom rules | 5 | 20 | 100 | 1000 | +| Rate limiting rules | 1 | 10 | 25 | 100 | +| Rule expression length | 4096 chars | 4096 chars | 4096 chars | 4096 chars | +| Rules per ruleset | 75 | 75 | 400 | 1000 | +| Managed rulesets | Yes | Yes | Yes | Yes | +| Rate limit characteristics | 2 | 3 | 5 | 5 | + +**Important Notes:** +- Rules execute in order; first match wins (except skip rules) +- Expression evaluation stops at first `false` in AND chains +- `matches` regex operator is slower than string operators +- Rate limit counting happens before mitigation + +## API Errors + +**Problem:** API calls fail with cryptic errors +**Cause:** Invalid parameters or permissions +**Solution:** + +```typescript +// Error: "Invalid phase" → Use exact phase name +phase: 'http_request_firewall_custom' + +// Error: "Ruleset already exists" → Use update() or list first +const rulesets = await client.rulesets.list({ zone_id, phase: 'http_request_firewall_custom' }); +if (rulesets.result.length > 0) { + await client.rulesets.update({ zone_id, ruleset_id: rulesets.result[0].id, rules: [...] }); +} + +// Error: "Action not supported" → Check phase/action compatibility +// 'execute' only in http_request_firewall_managed +// Rate limit config only in http_ratelimit phase + +// Error: "Expression parse error" → Common fixes: +'ip.geoip.country eq "US"' // Quote strings +'cf.waf.score gt 40' // Use 'gt' not '>' +'http.request.uri.path' // Not 'http.request.path' +``` + +**Tip**: Test expressions in dashboard Security Events before deploying. diff --git a/plugins/cloudflare/skills/cloudflare/references/waf/patterns.md b/plugins/cloudflare/skills/cloudflare/references/waf/patterns.md new file mode 100644 index 0000000..1fe0004 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/waf/patterns.md @@ -0,0 +1,197 @@ +# Common Patterns + +## Deploy Managed Rulesets + +```typescript +// Deploy Cloudflare Managed Ruleset (default) +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_request_firewall_managed', + name: 'Cloudflare Managed Ruleset', + rules: [{ + action: 'execute', + action_parameters: { + id: 'efb7b8c949ac4650a09736fc376e9aee', // Cloudflare Managed + // Or: '4814384a9e5d4991b9815dcfc25d2f1f' for OWASP CRS + // Or: 'c2e184081120413c86c3ab7e14069605' for Exposed Credentials + }, + expression: 'true', // All requests + // Or: 'http.request.uri.path starts_with "/api"' for specific paths + enabled: true, + }], +}); +``` + +## Override Managed Ruleset + +```typescript +await client.rulesets.create({ + zone_id: 'zone_id', + phase: 'http_request_firewall_managed', + rules: [{ + action: 'execute', + action_parameters: { + id: 'efb7b8c949ac4650a09736fc376e9aee', + overrides: { + // Override specific rules + rules: [ + { id: '5de7edfa648c4d6891dc3e7f84534ffa', action: 'log' }, + { id: '75a0060762034b9dad4e883afc121b4c', enabled: false }, + ], + // Override categories: wordpress, sqli, xss, rce, etc. + categories: [ + { category: 'wordpress', enabled: false }, + { category: 'sqli', action: 'log' }, + ], + }, + }, + expression: 'true', + }], +}); +``` + +## Custom Rules + +```typescript +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_request_firewall_custom', + name: 'Custom WAF Rules', + rules: [ + // Attack score-based + { action: 'block', expression: 'cf.waf.score gt 50', enabled: true }, + { action: 'challenge', expression: 'cf.waf.score gt 20', enabled: true }, + + // Specific attack types + { action: 'block', expression: 'cf.waf.score.sqli gt 30 or cf.waf.score.xss gt 30', enabled: true }, + + // Geographic blocking + { action: 'block', expression: 'ip.geoip.country in {"CN" "RU"}', enabled: true }, + ], +}); +``` + +## Rate Limiting + +```typescript +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_ratelimit', + name: 'Rate Limits', + rules: [ + // Per-IP global limit + { + action: 'block', + expression: 'true', + action_parameters: { + ratelimit: { + characteristics: ['cf.colo.id', 'ip.src'], + period: 60, + requests_per_period: 100, + mitigation_timeout: 600, + }, + }, + }, + + // Login endpoint (stricter) + { + action: 'block', + expression: 'http.request.uri.path eq "/api/login"', + action_parameters: { + ratelimit: { + characteristics: ['ip.src'], + period: 60, + requests_per_period: 5, + mitigation_timeout: 600, + }, + }, + }, + + // API writes only (using counting_expression) + { + action: 'block', + expression: 'http.request.uri.path starts_with "/api"', + action_parameters: { + ratelimit: { + characteristics: ['cf.colo.id', 'ip.src'], + period: 60, + requests_per_period: 50, + counting_expression: 'http.request.method ne "GET"', + }, + }, + }, + ], +}); +``` + +## Skip Rules + +```typescript +await client.rulesets.create({ + zone_id: 'zone_id', + kind: 'zone', + phase: 'http_request_firewall_custom', + name: 'Skip Rules', + rules: [ + // Skip static assets (current ruleset only) + { + action: 'skip', + action_parameters: { ruleset: 'current' }, + expression: 'http.request.uri.path matches "\\.(jpg|css|js|woff2?)$"', + }, + + // Skip all WAF phases for trusted IPs + { + action: 'skip', + action_parameters: { + phases: ['http_request_firewall_managed', 'http_ratelimit'], + }, + expression: 'ip.src in {192.0.2.0/24}', + }, + ], +}); +``` + +## Complete Setup Example + +Combine all three phases for comprehensive protection: + +```typescript +const client = new Cloudflare({ apiToken: process.env.CF_API_TOKEN }); +const zoneId = process.env.ZONE_ID; + +// 1. Custom rules (execute first) +await client.rulesets.create({ + zone_id: zoneId, + phase: 'http_request_firewall_custom', + rules: [ + { action: 'skip', action_parameters: { phases: ['http_request_firewall_managed', 'http_ratelimit'] }, expression: 'ip.src in {192.0.2.0/24}' }, + { action: 'block', expression: 'cf.waf.score gt 50' }, + { action: 'managed_challenge', expression: 'cf.waf.score gt 20' }, + ], +}); + +// 2. Managed ruleset (execute second) +await client.rulesets.create({ + zone_id: zoneId, + phase: 'http_request_firewall_managed', + rules: [{ + action: 'execute', + action_parameters: { id: 'efb7b8c949ac4650a09736fc376e9aee', overrides: { categories: [{ category: 'wordpress', enabled: false }] } }, + expression: 'true', + }], +}); + +// 3. Rate limiting (execute third) +await client.rulesets.create({ + zone_id: zoneId, + phase: 'http_ratelimit', + rules: [ + { action: 'block', expression: 'true', action_parameters: { ratelimit: { characteristics: ['cf.colo.id', 'ip.src'], period: 60, requests_per_period: 100, mitigation_timeout: 600 } } }, + { action: 'block', expression: 'http.request.uri.path eq "/api/login"', action_parameters: { ratelimit: { characteristics: ['ip.src'], period: 60, requests_per_period: 5, mitigation_timeout: 600 } } }, + ], +}); +``` \ No newline at end of file diff --git a/plugins/cloudflare/skills/cloudflare/references/web-analytics/README.md b/plugins/cloudflare/skills/cloudflare/references/web-analytics/README.md new file mode 100644 index 0000000..fcc1938 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/web-analytics/README.md @@ -0,0 +1,140 @@ +# Cloudflare Web Analytics + +Privacy-first web analytics providing Core Web Vitals, traffic metrics, and user insights without compromising visitor privacy. + +## Overview + +Cloudflare Web Analytics provides: +- **Core Web Vitals** - LCP, FID, CLS, INP, TTFB monitoring +- **Page views & visits** - Traffic patterns without cookies +- **Referrers & paths** - Traffic sources and popular pages +- **Device & browser data** - User agent breakdown +- **Geographic data** - Country-level visitor distribution +- **Privacy-first** - No cookies, fingerprinting, or PII collection +- **Free** - No cost, unlimited pageviews + +**Important:** Web Analytics is **dashboard-only**. No API exists for programmatic data access. + +## Quick Start Decision Tree + +``` +Is your site proxied through Cloudflare? +├─ YES → Use automatic injection (configuration.md) +│ ├─ Enable auto-injection in dashboard +│ └─ No code changes needed (unless Cache-Control: no-transform) +│ +└─ NO → Use manual beacon integration (integration.md) + ├─ Add JS snippet to HTML + ├─ Use spa: true for React/Vue/Next.js + └─ Configure CSP if needed +``` + +## Reading Order + +1. **[configuration.md](configuration.md)** - Setup for proxied vs non-proxied sites +2. **[integration.md](integration.md)** - Framework-specific beacon integration (React, Next.js, Vue, Nuxt, etc.) +3. **[patterns.md](patterns.md)** - Common use cases (performance monitoring, GDPR consent, multi-site tracking) +4. **[gotchas.md](gotchas.md)** - Troubleshooting (SPA tracking, CSP issues, hash routing limitations) + +## When to Use Each File + +- **Setting up for first time?** → Start with configuration.md +- **Using React/Next.js/Vue/Nuxt?** → Go to integration.md for framework code +- **Need GDPR consent loading?** → See patterns.md +- **Beacon not loading or no data?** → Check gotchas.md +- **SPA not tracking navigation?** → See integration.md for `spa: true` config + +## Key Concepts + +### Proxied vs Non-Proxied Sites + +| Type | Description | Beacon Injection | Limit | +|------|-------------|------------------|-------| +| **Proxied** | DNS through Cloudflare (orange cloud) | Automatic or manual | Unlimited | +| **Non-proxied** | External hosting, manual beacon | Manual only | 10 sites max | + +### SPA Mode + +**Critical for modern frameworks:** +```json +{"token": "YOUR_TOKEN", "spa": true} +``` + +Without `spa: true`, client-side navigation (React Router, Vue Router, Next.js routing) will NOT be tracked. Only initial page loads will register. + +### CSP Requirements + +If using Content Security Policy, allow both domains: +``` +script-src https://static.cloudflareinsights.com https://cloudflareinsights.com; +``` + +## Features + +### Core Web Vitals Debugging +- **LCP (Largest Contentful Paint)** - Identifies slow-loading hero images/elements +- **FID (First Input Delay)** - Interaction responsiveness (legacy metric) +- **INP (Interaction to Next Paint)** - Modern interaction responsiveness metric +- **CLS (Cumulative Layout Shift)** - Visual stability issues +- **TTFB (Time to First Byte)** - Server response performance + +Dashboard shows top 5 problematic elements with CSS selectors for debugging. + +### Traffic Filters +- **Bot filtering** - Exclude automated traffic from metrics +- **Date ranges** - Custom time period analysis +- **Geographic** - Country-level filtering +- **Device type** - Desktop, mobile, tablet breakdown +- **Browser/OS** - User agent filtering + +### Rules (Advanced - Plan-dependent) + +Create custom tracking rules for advanced configurations: + +**Sample Rate Rules:** +- Reduce data collection percentage for high-traffic sites +- Example: Track only 50% of visitors to reduce volume + +**Path-Based Rules:** +- Different behavior per route +- Example: Exclude `/admin/*` or `/internal/*` from tracking + +**Host-Based Rules:** +- Multi-domain configurations +- Example: Separate tracking for staging vs production subdomains + +**Availability:** Rules feature depends on your Cloudflare plan. Check dashboard under Web Analytics → Rules to see if available. Free plans may have limited or no access. + +## Plan Limits + +| Feature | Free | Notes | +|---------|------|-------| +| Proxied sites | Unlimited | DNS through Cloudflare | +| Non-proxied sites | 10 | External hosting | +| Pageviews | Unlimited | No volume limits | +| Data retention | 6 months | Rolling window | +| Rules | Plan-dependent | Check dashboard | + +## Privacy & Compliance + +- **No cookies** - Zero client-side storage +- **No fingerprinting** - No tracking across sites +- **No PII** - IP addresses not stored +- **GDPR-friendly** - Minimal data collection +- **CCPA-compliant** - No personal data sale + +**EU opt-out:** Dashboard option to exclude EU visitor data entirely. + +## Limitations + +- **Dashboard-only** - No API for programmatic access +- **No real-time** - 5-10 minute data delay +- **No custom events** - Automatic pageview/navigation tracking only +- **History API only** - Hash-based routing (`#/path`) not supported +- **No session replay** - Metrics only, no user recordings +- **No form tracking** - Page navigation tracking only + +## See Also + +- [Cloudflare Web Analytics Docs](https://developers.cloudflare.com/analytics/web-analytics/) +- [Core Web Vitals Guide](https://web.dev/vitals/) diff --git a/plugins/cloudflare/skills/cloudflare/references/web-analytics/configuration.md b/plugins/cloudflare/skills/cloudflare/references/web-analytics/configuration.md new file mode 100644 index 0000000..ff8f18d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/web-analytics/configuration.md @@ -0,0 +1,76 @@ +# Configuration + +## Setup Methods + +### Proxied Sites (Automatic) + +Dashboard → Web Analytics → Add site → Select hostname → Done + +| Injection Option | Description | +|------------------|-------------| +| Enable | Auto-inject for all visitors (default) | +| Enable, excluding EU | No injection for EU (GDPR) | +| Enable with manual snippet | You add beacon manually | +| Disable | Pause tracking | + +**Fails if response has:** `Cache-Control: public, no-transform` + +**CSP required:** +``` +script-src https://static.cloudflareinsights.com https://cloudflareinsights.com; +``` + +### Non-Proxied Sites (Manual) + +Dashboard → Web Analytics → Add site → Enter hostname → Copy snippet + +```html + +``` + +**Limits:** 10 non-proxied sites per account + +## SPA Mode + +**Enable `spa: true` for:** React Router, Next.js, Vue Router, Nuxt, SvelteKit, Angular + +**Keep `spa: false` for:** Traditional multi-page apps, static sites, WordPress + +**Hash routing (`#/path`) NOT supported** - use History API routing. + +## Token Management + +- Found in: Dashboard → Web Analytics → Manage site +- **Not secrets** - domain-locked, safe to expose in HTML +- Each site gets unique token + +## Environment Config + +```typescript +// Only load in production +if (process.env.NODE_ENV === 'production') { + // Load beacon +} +``` + +Or use environment-specific tokens via env vars. + +## Verify Installation + +1. DevTools Network → filter `cloudflareinsights` → see `beacon.min.js` + data request +2. No CSP/CORS errors in console +3. Dashboard shows pageviews after 5-10 min delay + +## Rules (Plan-dependent) + +Configure in dashboard for: +- **Sample rate** - reduce collection % for high-traffic +- **Path-based** - different behavior per route +- **Host-based** - separate tracking per domain + +## Data Retention + +- 6 months rolling window +- 1-hour bucket granularity +- No raw export, dashboard only diff --git a/plugins/cloudflare/skills/cloudflare/references/web-analytics/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/web-analytics/gotchas.md new file mode 100644 index 0000000..cad1424 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/web-analytics/gotchas.md @@ -0,0 +1,82 @@ +# Web Analytics Gotchas + +## Critical Issues + +### SPA Navigation Not Tracked + +**Symptom:** Only initial pageload counted +**Fix:** Add `spa: true`: +```html + +``` + +### CSP Blocking Beacon + +**Symptom:** Console error "Refused to load script" +**Fix:** Allow both domains: +``` +script-src 'self' https://static.cloudflareinsights.com https://cloudflareinsights.com; +``` + +### Hash-Based Routing Unsupported + +**Symptom:** `#/path` URLs not tracked +**Fix:** Migrate to History API (`BrowserRouter`, not `HashRouter`). No workaround for hash routing. + +### No Data Appearing + +**Causes & Fixes:** +1. **Delay** - Wait 5-15 minutes +2. **Wrong token** - Verify matches dashboard exactly +3. **Script blocked** - Check DevTools Network tab for beacon.min.js +4. **Domain mismatch** - Dashboard site must match actual URL + +### Auto-Injection Fails + +**Cause:** `Cache-Control: no-transform` header +**Fix:** Remove `no-transform` or install beacon manually + +### Duplicate Pageviews + +**Cause:** Multiple beacon scripts +**Fix:** Keep only one beacon per page + +## Configuration Issues + +| Issue | Fix | +|-------|-----| +| 10-site limit reached | Delete old sites or proxy through CF (unlimited) | +| Token not recognized | Use exact alphanumeric token from dashboard | + +## Framework-Specific + +### Next.js Hydration Warning + +```tsx + +``` + +Place before closing `` tag. + +## Framework Examples + +| Framework | Location | Notes | +|-----------|----------|-------| +| React/Vite | `public/index.html` | Add `spa: true` | +| Next.js App Router | `app/layout.tsx` | Use ` +``` + +Without `spa: true`: only initial pageload tracked. + +## Staging/Production Separation + +```typescript +// Use env-specific tokens +const token = process.env.NEXT_PUBLIC_CF_ANALYTICS_TOKEN; +// .env.production: production token +// .env.staging: staging token (or empty to disable) +``` + +## Bot Filtering + +Dashboard → Filters → "Exclude Bot Traffic" + +Filters: Search crawlers, monitoring services, known bots. +Not filtered: Headless browsers (Playwright/Puppeteer). + +## Ad-Blocker Impact + +~25-40% of users may block `cloudflareinsights.com`. No official workaround. +Dashboard shows minimum baseline; use server logs for complete picture. + +## Limitations + +- No UTM parameter tracking +- No webhooks/alerts/API +- No custom beacon domains +- Max 10 non-proxied sites diff --git a/plugins/cloudflare/skills/cloudflare/references/workerd/README.md b/plugins/cloudflare/skills/cloudflare/references/workerd/README.md new file mode 100644 index 0000000..11b2c6d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workerd/README.md @@ -0,0 +1,78 @@ +# Workerd Runtime + +V8-based JS/Wasm runtime powering Cloudflare Workers. Use as app server, dev tool, or HTTP proxy. + +## ⚠️ IMPORTANT SECURITY NOTICE +**workerd is NOT a hardened sandbox.** Do not run untrusted code. It's designed for deploying YOUR code locally/self-hosted, not multi-tenant SaaS. Cloudflare production adds security layers not present in open-source workerd. + +## Decision Tree: When to Use What + +**95% of users:** Use Wrangler +- Local development: `wrangler dev` (uses workerd internally) +- Deployment: `wrangler deploy` (deploys to Cloudflare) +- Types: `wrangler types` (generates TypeScript types) + +**Use raw workerd directly only if:** +- Self-hosting Workers runtime in production +- Embedding runtime in C++ application +- Custom tooling/testing infrastructure +- Debugging workerd-specific behavior + +**Never use workerd for:** +- Running untrusted/user-submitted code +- Multi-tenant isolation (not hardened) +- Production without additional security layers + +## Key Features +- **Standards-based**: Fetch API, Web Crypto, Streams, WebSocket +- **Nanoservices**: Service bindings with local call performance +- **Capability security**: Explicit bindings prevent SSRF +- **Backwards compatible**: Version = max compat date supported + +## Architecture +``` +Config (workerd.capnp) +├── Services (workers/endpoints) +├── Sockets (HTTP/HTTPS listeners) +└── Extensions (global capabilities) +``` + +## Quick Start +```bash +workerd serve config.capnp +workerd compile config.capnp myConfig -o binary +workerd test config.capnp +``` + +## Platform Support & Beta Status + +| Platform | Status | Notes | +|----------|--------|-------| +| Linux (x64) | Stable | Primary platform | +| macOS (x64/ARM) | Stable | Full support | +| Windows | Beta | Use WSL2 for best results | +| Linux (ARM64) | Experimental | Limited testing | + +workerd is in **active development**. Breaking changes possible. Pin versions in production. + +## Core Concepts +- **Service**: Named endpoint (worker/network/disk/external) +- **Binding**: Capability-based resource access (KV/DO/R2/services) +- **Compatibility date**: Feature gate (always set!) +- **Modules**: ES modules (recommended) or service worker syntax + +## Reading Order (Progressive Disclosure) + +**Start here:** +1. This README (overview, decision tree) +2. [patterns.md](./patterns.md) - Common workflows, framework examples + +**When you need details:** +3. [configuration.md](./configuration.md) - Config format, services, bindings +4. [api.md](./api.md) - Runtime APIs, TypeScript types +5. [gotchas.md](./gotchas.md) - Common errors, debugging + +## Related References +- [workers](../workers/) - Workers runtime API documentation +- [miniflare](../miniflare/) - Testing tool built on workerd +- [wrangler](../wrangler/) - CLI that uses workerd for local dev diff --git a/plugins/cloudflare/skills/cloudflare/references/workerd/api.md b/plugins/cloudflare/skills/cloudflare/references/workerd/api.md new file mode 100644 index 0000000..085f507 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workerd/api.md @@ -0,0 +1,185 @@ +# Workerd APIs + +## Worker Code (JS/TS) + +### ES Modules (Recommended) +```javascript +export default { + async fetch(request, env, ctx) { + const value = await env.KV.get("key"); // Bindings in env + const response = await env.API.fetch(request); // Service binding + ctx.waitUntil(logRequest(request)); // Background task + return new Response("OK"); + }, + async adminApi(request, env, ctx) { /* Named entrypoint */ }, + async queue(batch, env, ctx) { /* Queue consumer */ }, + async scheduled(event, env, ctx) { /* Cron handler */ } +}; +``` + +### TypeScript Types + +**Generate from wrangler.toml (Recommended):** +```bash +wrangler types # Output: worker-configuration.d.ts +``` + +**Manual types:** +```typescript +interface Env { + API: Fetcher; + CACHE: KVNamespace; + STORAGE: R2Bucket; + ROOMS: DurableObjectNamespace; + API_KEY: string; +} + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + return new Response(await env.CACHE.get("key")); + } +}; +``` + +**Setup:** +```bash +npm install -D @cloudflare/workers-types +``` + +```json +// tsconfig.json +{"compilerOptions": {"types": ["@cloudflare/workers-types"]}} +``` + +### Service Worker Syntax (Legacy) +```javascript +addEventListener('fetch', event => { + event.respondWith(handleRequest(event.request)); +}); + +async function handleRequest(request) { + const value = await KV.get("key"); // Bindings as globals + return new Response("OK"); +} +``` + +### Durable Objects +```javascript +export class Room { + constructor(state, env) { this.state = state; this.env = env; } + + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/increment") { + const value = (await this.state.storage.get("counter")) || 0; + await this.state.storage.put("counter", value + 1); + return new Response(String(value + 1)); + } + return new Response("Not found", {status: 404}); + } +} +``` + +### RPC Between Services +```javascript +// Caller: env.AUTH.validateToken(token) returns structured data +const user = await env.AUTH.validateToken(request.headers.get("Authorization")); + +// Callee: export methods that return data +export default { + async validateToken(token) { return {id: 123, name: "Alice"}; } +}; +``` + +## Web Platform APIs + +### Fetch +- `fetch()`, `Request`, `Response`, `Headers` +- `AbortController`, `AbortSignal` + +### Streams +- `ReadableStream`, `WritableStream`, `TransformStream` +- Byte streams, BYOB readers + +### Web Crypto +- `crypto.subtle` (encrypt/decrypt/sign/verify) +- `crypto.randomUUID()`, `crypto.getRandomValues()` + +### Encoding +- `TextEncoder`, `TextDecoder` +- `atob()`, `btoa()` + +### Web Standards +- `URL`, `URLSearchParams` +- `Blob`, `File`, `FormData` +- `WebSocket` + +### Server-Sent Events (EventSource) +```javascript +// Server-side SSE +const { readable, writable } = new TransformStream(); +const writer = writable.getWriter(); +writer.write(new TextEncoder().encode('data: Hello\n\n')); +return new Response(readable, {headers: {'Content-Type': 'text/event-stream'}}); +``` + +### HTMLRewriter (HTML Parsing/Transformation) +```javascript +const response = await fetch('https://example.com'); +return new HTMLRewriter() + .on('a[href]', { + element(el) { + el.setAttribute('href', `/proxy?url=${encodeURIComponent(el.getAttribute('href'))}`); + } + }) + .on('script', { element(el) { el.remove(); } }) + .transform(response); +``` + +### TCP Sockets (Experimental) +```javascript +const socket = await connect({ hostname: 'example.com', port: 80 }); +const writer = socket.writable.getWriter(); +await writer.write(new TextEncoder().encode('GET / HTTP/1.1\r\n\r\n')); +const reader = socket.readable.getReader(); +const { value } = await reader.read(); +return new Response(value); +``` + +### Performance +- `performance.now()`, `performance.timeOrigin` +- `setTimeout()`, `setInterval()`, `queueMicrotask()` + +### Console +- `console.log()`, `console.error()`, `console.warn()` + +### Node.js Compat (`nodejs_compat` flag) +```javascript +import { Buffer } from 'node:buffer'; +import { randomBytes } from 'node:crypto'; + +const buf = Buffer.from('Hello'); +const random = randomBytes(16); +``` + +**Available:** `node:buffer`, `node:crypto`, `node:stream`, `node:util`, `node:events`, `node:assert`, `node:path`, `node:querystring`, `node:url` +**NOT available:** `node:fs`, `node:http`, `node:net`, `node:child_process` + +## CLI Commands + +```bash +workerd serve config.capnp [constantName] # Start server +workerd serve config.capnp --socket-addr http=*:3000 --verbose +workerd compile config.capnp constantName -o binary # Compile to binary +workerd test config.capnp [--test-only=test.js] # Run tests +``` + +## Wrangler Integration + +Use Wrangler for development: +```bash +wrangler dev # Uses workerd internally +wrangler types # Generate TypeScript types from wrangler.toml +``` + +See [patterns.md](./patterns.md) for usage examples, [configuration.md](./configuration.md) for config details. diff --git a/plugins/cloudflare/skills/cloudflare/references/workerd/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workerd/configuration.md new file mode 100644 index 0000000..bad5f43 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workerd/configuration.md @@ -0,0 +1,183 @@ +# Workerd Configuration + +## Basic Structure +```capnp +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [(name = "main", worker = .mainWorker)], + sockets = [(name = "http", address = "*:8080", http = (), service = "main")] +); + +const mainWorker :Workerd.Worker = ( + modules = [(name = "index.js", esModule = embed "src/index.js")], + compatibilityDate = "2024-01-15", + bindings = [...] +); +``` + +## Services +**Worker**: Run JS/Wasm code +```capnp +(name = "api", worker = ( + modules = [(name = "index.js", esModule = embed "index.js")], + compatibilityDate = "2024-01-15", + bindings = [...] +)) +``` + +**Network**: Internet access +```capnp +(name = "internet", network = (allow = ["public"], tlsOptions = (trustBrowserCas = true))) +``` + +**External**: Reverse proxy +```capnp +(name = "backend", external = (address = "api.com:443", http = (style = tls))) +``` + +**Disk**: Static files +```capnp +(name = "assets", disk = (path = "/var/www", writable = false)) +``` + +## Sockets +```capnp +(name = "http", address = "*:8080", http = (), service = "main") +(name = "https", address = "*:443", https = (options = (), tlsOptions = (keypair = (...))), service = "main") +(name = "app", address = "unix:/tmp/app.sock", http = (), service = "main") +``` + +## Worker Formats +```capnp +# ES Modules (recommended) +modules = [(name = "index.js", esModule = embed "src/index.js"), (name = "wasm.wasm", wasm = embed "build/module.wasm")] + +# Service Worker (legacy) +serviceWorkerScript = embed "worker.js" + +# CommonJS +(name = "legacy.js", commonJsModule = embed "legacy.js", namedExports = ["foo"]) +``` + +## Bindings +Bindings expose resources to workers. ES modules: `env.BINDING`, Service workers: globals. + +### Primitive Types +```capnp +(name = "API_KEY", text = "secret") # String +(name = "CONFIG", json = '{"key":"val"}') # Parsed JSON +(name = "DATA", data = embed "data.bin") # ArrayBuffer +(name = "DATABASE_URL", fromEnvironment = "DB_URL") # System env var +``` + +### Service Binding +```capnp +(name = "AUTH", service = "auth-worker") # Basic +(name = "API", service = ( + name = "backend", + entrypoint = "adminApi", # Named export + props = (json = '{"role":"admin"}') # ctx.props +)) +``` + +### Storage +```capnp +(name = "CACHE", kvNamespace = "kv-service") # KV +(name = "STORAGE", r2Bucket = "r2-service") # R2 +(name = "ROOMS", durableObjectNamespace = ( + serviceName = "room-service", + className = "Room" +)) +(name = "FAST", memoryCache = ( + id = "cache-id", + limits = (maxKeys = 1000, maxValueSize = 1048576) +)) +``` + +### Other +```capnp +(name = "TASKS", queue = "queue-service") +(name = "ANALYTICS", analyticsEngine = "analytics") +(name = "LOADER", workerLoader = (id = "dynamic")) +(name = "KEY", cryptoKey = (format = raw, algorithm = (name = "HMAC", hash = "SHA-256"), keyData = embed "key.bin", usages = [sign, verify], extractable = false)) +(name = "TRACED", wrapped = (moduleName = "tracing", entrypoint = "makeTracer", innerBindings = [(name = "backend", service = "backend")])) +``` + +## Compatibility +```capnp +compatibilityDate = "2024-01-15" # Always set! +compatibilityFlags = ["nodejs_compat", "streams_enable_constructors"] +``` + +Version = max compat date. Update carefully after testing. + +## Parameter Bindings (Inheritance) +```capnp +const base :Workerd.Worker = ( + modules = [...], compatibilityDate = "2024-01-15", + bindings = [(name = "API_URL", parameter = (type = text)), (name = "DB", parameter = (type = service))] +); + +const derived :Workerd.Worker = ( + inherit = "base-service", + bindings = [(name = "API_URL", text = "https://api.com"), (name = "DB", service = "postgres")] +); +``` + +## Durable Objects Config +```capnp +const worker :Workerd.Worker = ( + modules = [...], + compatibilityDate = "2024-01-15", + bindings = [(name = "ROOMS", durableObjectNamespace = "Room")], + durableObjectNamespaces = [(className = "Room", uniqueKey = "v1")], + durableObjectStorage = (localDisk = "/var/do") +); +``` + +## Remote Bindings (Development) + +Connect local workerd to production Cloudflare resources: + +```capnp +bindings = [ + # Remote KV (requires API token) + (name = "PROD_KV", kvNamespace = ( + remote = ( + accountId = "your-account-id", + namespaceId = "your-namespace-id", + apiToken = .envVar("CF_API_TOKEN") + ) + )), + + # Remote R2 + (name = "PROD_R2", r2Bucket = ( + remote = ( + accountId = "your-account-id", + bucketName = "my-bucket", + apiToken = .envVar("CF_API_TOKEN") + ) + )), + + # Remote Durable Object + (name = "PROD_DO", durableObjectNamespace = ( + remote = ( + accountId = "your-account-id", + scriptName = "my-worker", + className = "MyDO", + apiToken = .envVar("CF_API_TOKEN") + ) + )) +] +``` + +**Note:** Remote bindings require network access and valid Cloudflare API credentials. + +## Logging & Debugging +```capnp +logging = (structuredLogging = true, stdoutPrefix = "OUT: ", stderrPrefix = "ERR: ") +v8Flags = ["--expose-gc", "--max-old-space-size=2048"] # ⚠️ Unsupported in production +``` + +See [patterns.md](./patterns.md) for multi-service examples, [gotchas.md](./gotchas.md) for config errors. diff --git a/plugins/cloudflare/skills/cloudflare/references/workerd/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workerd/gotchas.md new file mode 100644 index 0000000..dc35109 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workerd/gotchas.md @@ -0,0 +1,139 @@ +# Workerd Gotchas + +## Common Errors + +### "Missing compatibility date" +**Cause:** Compatibility date not set +**Solution:** +❌ Wrong: +```capnp +const worker :Workerd.Worker = ( + serviceWorkerScript = embed "worker.js" +) +``` + +✅ Correct: +```capnp +const worker :Workerd.Worker = ( + serviceWorkerScript = embed "worker.js", + compatibilityDate = "2024-01-15" # Always set! +) +``` + +### Wrong Binding Type +**Problem:** JSON not parsed +**Cause:** Using `text = '{"key":"value"}'` instead of `json` +**Solution:** Use `json = '{"key":"value"}'` for parsed objects + +### Service vs Namespace +**Problem:** Cannot create DO instance +**Cause:** Using `service = "room-service"` for Durable Object +**Solution:** Use `durableObjectNamespace = "Room"` for DO bindings + +### Module Name Mismatch +**Problem:** Import fails +**Cause:** Module name includes path: `name = "src/index.js"` +**Solution:** Use simple names: `name = "index.js"`, embed with path + +## Network Access + +**Problem:** Fetch fails with network error +**Cause:** No network service configured (workerd has no global fetch) +**Solution:** Add network service binding: +```capnp +services = [(name = "internet", network = (allow = ["public"]))] +bindings = [(name = "NET", service = "internet")] +``` + +Or external service: +```capnp +bindings = [(name = "API", service = (external = (address = "api.com:443", http = (style = tls))))] +``` + +### "Worker not responding" +**Cause:** Socket misconfigured, no fetch handler, or port unavailable +**Solution:** Verify socket `address` matches, worker exports `fetch()`, port available + +### "Binding not found" +**Cause:** Name mismatch or service doesn't exist +**Solution:** Check binding name in config matches code (`env.BINDING` for ES modules) + +### "Module not found" +**Cause:** Module name doesn't match import or bad embed path +**Solution:** Module `name` must match import path exactly, verify `embed` path + +### "Compatibility error" +**Cause:** Date not set or API unavailable on that date +**Solution:** Set `compatibilityDate`, verify API available on that date + +## Performance Issues + +**Problem:** High memory usage +**Cause:** Large caches or many isolates +**Solution:** Set cache limits, reduce isolate count, or use V8 flags (caution) + +**Problem:** Slow startup +**Cause:** Many modules or complex config +**Solution:** Compile to binary (`workerd compile`), reduce imports + +**Problem:** Request timeouts +**Cause:** External service issues or DNS problems +**Solution:** Check connectivity, DNS resolution, TLS handshake + +## Build Issues + +**Problem:** Cap'n Proto syntax errors +**Cause:** Invalid config or missing schema +**Solution:** Install capnproto tools, validate: `capnp compile -I. config.capnp` + +**Problem:** Embed path not found +**Cause:** Path relative to config file +**Solution:** Use correct relative path or absolute path + +**Problem:** V8 flags cause crashes +**Cause:** Unsafe V8 flags +**Solution:** ⚠️ V8 flags unsupported in production. Test thoroughly before use. + +## Security Issues + +**Problem:** Hardcoded secrets in config +**Cause:** `text` binding with secret value +**Solution:** Use `fromEnvironment` to load from env vars + +**Problem:** Overly broad network access +**Cause:** `network = (allow = ["*"])` +**Solution:** Restrict to `allow = ["public"]` or specific hosts + +**Problem:** Extractable crypto keys +**Cause:** `cryptoKey = (extractable = true, ...)` +**Solution:** Set `extractable = false` unless export required + +## Compatibility Changes + +**Problem:** Breaking changes after compat date update +**Cause:** New flags enabled between dates +**Solution:** Review [compat dates docs](https://developers.cloudflare.com/workers/configuration/compatibility-dates/), test locally first + +**Problem:** "Compatibility date not supported" +**Cause:** Workerd version older than compat date +**Solution:** Update workerd binary (version = max compat date supported) + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| V8 flags | Unsupported in production | Use with caution | +| Compatibility date | Must match workerd version | Update if mismatch | +| Module count | Affects startup time | Many imports slow | + +## Troubleshooting Steps + +1. **Enable verbose logging**: `workerd serve config.capnp --verbose` +2. **Check logs**: Look for error messages, stack traces +3. **Validate config**: `capnp compile -I. config.capnp` +4. **Test bindings**: Log `Object.keys(env)` to verify +5. **Check versions**: Workerd version vs compat date +6. **Isolate issue**: Minimal repro config +7. **Review schema**: [workerd.capnp](https://github.com/cloudflare/workerd/blob/main/src/workerd/server/workerd.capnp) + +See [configuration.md](./configuration.md) for config details, [patterns.md](./patterns.md) for working examples, [api.md](./api.md) for runtime APIs. diff --git a/plugins/cloudflare/skills/cloudflare/references/workerd/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workerd/patterns.md new file mode 100644 index 0000000..5aaf092 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workerd/patterns.md @@ -0,0 +1,192 @@ +# Workerd Patterns + +## Multi-Service Architecture +```capnp +const config :Workerd.Config = ( + services = [ + (name = "frontend", worker = ( + modules = [(name = "index.js", esModule = embed "frontend/index.js")], + compatibilityDate = "2024-01-15", + bindings = [(name = "API", service = "api")] + )), + (name = "api", worker = ( + modules = [(name = "index.js", esModule = embed "api/index.js")], + compatibilityDate = "2024-01-15", + bindings = [(name = "DB", service = "postgres"), (name = "CACHE", kvNamespace = "kv")] + )), + (name = "postgres", external = (address = "db.internal:5432", http = ())), + (name = "kv", disk = (path = "/var/kv", writable = true)) + ], + sockets = [(name = "http", address = "*:8080", http = (), service = "frontend")] +); +``` + +## Durable Objects +```capnp +const worker :Workerd.Worker = ( + modules = [(name = "index.js", esModule = embed "index.js"), (name = "room.js", esModule = embed "room.js")], + compatibilityDate = "2024-01-15", + bindings = [(name = "ROOMS", durableObjectNamespace = "Room")], + durableObjectNamespaces = [(className = "Room", uniqueKey = "v1")], + durableObjectStorage = (localDisk = "/var/do") +); +``` + +## Dev vs Prod Configs +```capnp +# Use parameter bindings for env-specific config +const baseWorker :Workerd.Worker = ( + modules = [(name = "index.js", esModule = embed "src/index.js")], + compatibilityDate = "2024-01-15", + bindings = [(name = "API_URL", parameter = (type = text))] +); + +const prodWorker :Workerd.Worker = ( + inherit = "base-service", + bindings = [(name = "API_URL", text = "https://api.prod.com")] +); +``` + +## HTTP Reverse Proxy +```capnp +services = [ + (name = "proxy", worker = (serviceWorkerScript = embed "proxy.js", compatibilityDate = "2024-01-15", bindings = [(name = "BACKEND", service = "backend")])), + (name = "backend", external = (address = "internal:8080", http = ())) +] +``` + +## Local Development + +**Recommended:** Use Wrangler +```bash +wrangler dev # Uses workerd internally +``` + +**Direct workerd:** +```bash +workerd serve config.capnp --socket-addr http=*:3000 --verbose +``` + +**Environment variables:** +```capnp +bindings = [(name = "DATABASE_URL", fromEnvironment = "DATABASE_URL")] +``` + +## Testing +```bash +workerd test config.capnp +workerd test config.capnp --test-only=test.js +``` + +Test files must be included in `modules = [...]` config. + +## Production Deployment + +### Compiled Binary (Recommended) +```bash +workerd compile config.capnp myConfig -o production-server +./production-server +``` + +### Docker +```dockerfile +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y ca-certificates +COPY workerd /usr/local/bin/ +COPY config.capnp /etc/workerd/ +COPY src/ /etc/workerd/src/ +EXPOSE 8080 +CMD ["workerd", "serve", "/etc/workerd/config.capnp"] +``` + +### Systemd +```ini +# /etc/systemd/system/workerd.service +[Service] +ExecStart=/usr/bin/workerd serve /etc/workerd/config.capnp --socket-fd http=3 +Restart=always +User=nobody +``` + +See systemd socket activation docs for complete setup. + +## Framework Integration + +### Hono +```javascript +import { Hono } from 'hono'; + +const app = new Hono(); + +app.get('/', (c) => c.text('Hello Hono!')); +app.get('/api/:id', async (c) => { + const id = c.req.param('id'); + const data = await c.env.KV.get(id); + return c.json({ id, data }); +}); + +export default app; +``` + +### itty-router +```javascript +import { Router } from 'itty-router'; + +const router = Router(); + +router.get('/', () => new Response('Hello itty!')); +router.get('/api/:id', async (request, env) => { + const { id } = request.params; + const data = await env.KV.get(id); + return Response.json({ id, data }); +}); + +export default { + fetch: (request, env, ctx) => router.handle(request, env, ctx) +}; +``` + +## Best Practices + +1. **Use ES modules** over service worker syntax +2. **Explicit bindings** - no global namespace assumptions +3. **Type safety** - define `Env` interfaces (use `wrangler types`) +4. **Service isolation** - split concerns into multiple services +5. **Pin compat date** in production after testing +6. **Use ctx.waitUntil()** for background tasks +7. **Handle errors gracefully** with try/catch +8. **Configure resource limits** on caches/storage + +## Common Patterns + +### Error Handling +```javascript +export default { + async fetch(request, env, ctx) { + try { + return await handleRequest(request, env); + } catch (error) { + console.error("Request failed", error); + return new Response("Internal Error", {status: 500}); + } + } +}; +``` + +### Background Tasks +```javascript +export default { + async fetch(request, env, ctx) { + const response = new Response("OK"); + + // Fire-and-forget background work + ctx.waitUntil( + env.ANALYTICS.put(request.url, Date.now()) + ); + + return response; + } +}; +``` + +See [configuration.md](./configuration.md) for config syntax, [api.md](./api.md) for runtime APIs, [gotchas.md](./gotchas.md) for common errors. diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-ai/README.md b/plugins/cloudflare/skills/cloudflare/references/workers-ai/README.md new file mode 100644 index 0000000..a8419d2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-ai/README.md @@ -0,0 +1,197 @@ +# Cloudflare Workers AI + +Expert guidance for Cloudflare Workers AI - serverless GPU-powered AI inference at the edge. + +## Overview + +Workers AI provides: +- 50+ pre-trained models (LLMs, embeddings, image generation, speech-to-text, translation) +- Native Workers binding (no external API calls) +- Pay-per-use pricing (neurons consumed per inference) +- OpenAI-compatible REST API +- Streaming support for text generation +- Function calling with compatible models + +**Architecture**: Inference runs on Cloudflare's GPU network. Models load on first request (cold start 1-3s), subsequent requests are faster. + +## Quick Start + +```typescript +interface Env { + AI: Ai; +} + +export default { + async fetch(request: Request, env: Env) { + const response = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages: [{ role: 'user', content: 'What is Cloudflare?' }] + }); + return Response.json(response); + } +}; +``` + +```bash +# Setup - add binding to wrangler.jsonc +wrangler dev --remote # Must use --remote for AI +wrangler deploy +``` + +## Model Selection Decision Tree + +### Text Generation (Chat/Completion) + +**Quality Priority**: +- **Best quality**: `@cf/meta/llama-3.1-70b-instruct` (expensive, ~2000 neurons) +- **Balanced**: `@cf/meta/llama-3.1-8b-instruct` (good quality, ~200 neurons) +- **Fastest/cheapest**: `@cf/mistral/mistral-7b-instruct-v0.1` (~50 neurons) + +**Function Calling**: +- Use `@cf/meta/llama-3.1-8b-instruct` or `@cf/meta/llama-3.1-70b-instruct` (native tool support) + +**Code Generation**: +- Use `@cf/deepseek-ai/deepseek-coder-6.7b-instruct` (specialized for code) + +### Embeddings (Semantic Search/RAG) + +**English text**: +- **Best**: `@cf/baai/bge-large-en-v1.5` (1024 dims, highest quality) +- **Balanced**: `@cf/baai/bge-base-en-v1.5` (768 dims, good quality) +- **Fast**: `@cf/baai/bge-small-en-v1.5` (384 dims, lower quality but fast) + +**Multilingual**: +- Use `@hf/sentence-transformers/paraphrase-multilingual-minilm-l12-v2` + +### Image Generation + +- **Stable Diffusion**: `@cf/stabilityai/stable-diffusion-xl-base-1.0` (~10,000 neurons) +- **Portraits**: `@cf/lykon/dreamshaper-8-lcm` (optimized for faces) + +### Other Tasks + +- **Speech-to-text**: `@cf/openai/whisper` +- **Translation**: `@cf/meta/m2m100-1.2b` (100 languages) +- **Image classification**: `@cf/microsoft/resnet-50` + +## SDK Approach Decision Tree + +### Native Binding (Recommended) + +**When**: Building Workers/Pages with TypeScript +**Why**: Zero external dependencies, best performance, native types + +```typescript +await env.AI.run(model, input); +``` + +### REST API + +**When**: External services, non-Workers environments, testing +**Why**: Standard HTTP, works anywhere + +```bash +curl https://api.cloudflare.com/client/v4/accounts//ai/run/@cf/meta/llama-3.1-8b-instruct \ + -H "Authorization: Bearer " \ + -d '{"messages":[{"role":"user","content":"Hello"}]}' +``` + +### Vercel AI SDK Integration + +**When**: Using Vercel AI SDK features (streaming UI, tool calling abstractions) +**Why**: Unified interface across providers + +```typescript +import { openai } from '@ai-sdk/openai'; + +const model = openai('model-name', { + baseURL: 'https://api.cloudflare.com/client/v4/accounts//ai/v1', + headers: { Authorization: 'Bearer ' } +}); +``` + +## RAG vs Direct Generation + +### Use RAG (Vectorize + Workers AI) When: +- Answering questions about specific documents/data +- Need factual accuracy from known corpus +- Context exceeds model's window (>4K tokens) +- Building knowledge base chat + +### Use Direct Generation When: +- Creative writing, brainstorming +- General knowledge questions +- Small context fits in prompt (<4K tokens) +- Cost optimization (RAG adds embedding + vector search costs) + +## Platform Limits + +| Limit | Free Tier | Paid Plans | +|-------|-----------|------------| +| Neurons/day | 10,000 | Pay per use | +| Rate limit | Varies by model | Higher (contact support) | +| Context window | Model dependent (2K-8K) | Same | +| Streaming | ✅ Supported | ✅ Supported | +| Function calling | ✅ Supported (select models) | ✅ Supported | + +**Pricing**: Free 10K neurons/day, then pay per neuron consumed (varies by model) + +## Common Tasks + +```typescript +// Streaming text generation +const stream = await env.AI.run(model, { messages, stream: true }); +for await (const chunk of stream) { + console.log(chunk.response); +} + +// Embeddings for RAG +const { data } = await env.AI.run('@cf/baai/bge-base-en-v1.5', { + text: ['Query text', 'Document 1', 'Document 2'] +}); + +// Function calling +const response = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages: [{ role: 'user', content: 'What is the weather?' }], + tools: [{ + type: 'function', + function: { name: 'getWeather', parameters: { ... } } + }] +}); +``` + +## Development Workflow + +```bash +# Always use --remote for AI (local doesn't have models) +wrangler dev --remote + +# Deploy to production +wrangler deploy + +# View model catalog +# https://developers.cloudflare.com/workers-ai/models/ +``` + +## Reading Order + +**Start here**: Quick Start above → configuration.md (setup) + +**Common tasks**: +- First time setup: configuration.md → Add binding + deploy +- Choose model: Model Selection Decision Tree (above) → api.md +- Build RAG: patterns.md → Vectorize integration +- Optimize costs: Model Selection + gotchas.md (rate limits) +- Debugging: gotchas.md → Common errors + +## In This Reference + +- [configuration.md](./configuration.md) - wrangler.jsonc setup, TypeScript types, bindings, environment variables +- [api.md](./api.md) - env.AI.run(), streaming, function calling, REST API, response types +- [patterns.md](./patterns.md) - RAG with Vectorize, prompt engineering, batching, error handling, caching +- [gotchas.md](./gotchas.md) - Deprecated @cloudflare/ai package, rate limits, pricing, common errors + +## See Also + +- [vectorize](../vectorize/) - Vector database for RAG patterns +- [ai-gateway](../ai-gateway/) - Caching, rate limiting, analytics for AI requests +- [workers](../workers/) - Worker runtime and fetch handler patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-ai/api.md b/plugins/cloudflare/skills/cloudflare/references/workers-ai/api.md new file mode 100644 index 0000000..e65f97a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-ai/api.md @@ -0,0 +1,112 @@ +# Workers AI API Reference + +## Core Method + +```typescript +const response = await env.AI.run(model, input); +``` + +## Text Generation + +```typescript +const result = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages: [ + { role: 'system', content: 'You are helpful' }, + { role: 'user', content: 'Hello' } + ], + temperature: 0.7, // 0-1 + max_tokens: 100 +}); +console.log(result.response); +``` + +**Streaming:** +```typescript +const stream = await env.AI.run(model, { messages, stream: true }); +return new Response(stream, { headers: { 'Content-Type': 'text/event-stream' } }); +``` + +## Embeddings + +```typescript +const result = await env.AI.run('@cf/baai/bge-base-en-v1.5', { + text: ['Query', 'Doc 1', 'Doc 2'] // Batch for efficiency +}); +const [queryEmbed, doc1Embed, doc2Embed] = result.data; // 768-dim vectors +``` + +## Function Calling + +```typescript +const tools = [{ + type: 'function', + function: { + name: 'getWeather', + description: 'Get weather for location', + parameters: { + type: 'object', + properties: { location: { type: 'string' } }, + required: ['location'] + } + } +}]; + +const response = await env.AI.run(model, { messages, tools }); +if (response.tool_calls) { + const args = JSON.parse(response.tool_calls[0].function.arguments); + // Execute function, send result back +} +``` + +## Image Generation + +```typescript +const image = await env.AI.run('@cf/stabilityai/stable-diffusion-xl-base-1.0', { + prompt: 'Mountain sunset', + num_steps: 20, // 1-20 + guidance: 7.5 // 1-20 +}); +return new Response(image, { headers: { 'Content-Type': 'image/png' } }); +``` + +## Speech Recognition + +```typescript +const audioArray = Array.from(new Uint8Array(await request.arrayBuffer())); +const result = await env.AI.run('@cf/openai/whisper', { audio: audioArray }); +console.log(result.text); +``` + +## Translation + +```typescript +const result = await env.AI.run('@cf/meta/m2m100-1.2b', { + text: 'Hello', + source_lang: 'en', + target_lang: 'es' +}); +console.log(result.translated_text); +``` + +## REST API + +```bash +curl https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/@cf/meta/llama-3.1-8b-instruct \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"messages":[{"role":"user","content":"Hello"}]}' +``` + +## Error Codes + +| Code | Meaning | Fix | +|------|---------|-----| +| 7502 | Model not found | Check spelling | +| 7504 | Validation failed | Verify input schema | +| 7505 | Rate limited | Reduce rate or upgrade | +| 7506 | Context exceeded | Reduce input size | + +## Performance Tips + +1. **Batch embeddings** - single request for multiple texts +2. **Stream long responses** - reduce perceived latency +3. **Accept cold starts** - first request ~1-3s, subsequent ~100-500ms diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-ai/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workers-ai/configuration.md new file mode 100644 index 0000000..f5563b3 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-ai/configuration.md @@ -0,0 +1,97 @@ +# Workers AI Configuration + +## wrangler.jsonc + +```jsonc +{ + "name": "my-ai-worker", + "main": "src/index.ts", + "compatibility_date": "2024-01-01", + "ai": { + "binding": "AI" + } +} +``` + +## TypeScript + +```bash +npm install --save-dev @cloudflare/workers-types +``` + +```typescript +interface Env { + AI: Ai; +} + +export default { + async fetch(request: Request, env: Env) { + const response = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages: [{ role: 'user', content: 'Hello' }] + }); + return Response.json(response); + } +}; +``` + +## Local Development + +```bash +wrangler dev --remote # Required for AI - no local inference +``` + +## REST API + +```typescript +const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/ai/run/@cf/meta/llama-3.1-8b-instruct`, + { + method: 'POST', + headers: { 'Authorization': `Bearer ${API_TOKEN}` }, + body: JSON.stringify({ messages: [{ role: 'user', content: 'Hello' }] }) + } +); +``` + +Create API token at: dash.cloudflare.com/profile/api-tokens (Workers AI - Read permission) + +## SDK Compatibility + +**OpenAI SDK:** +```typescript +import OpenAI from 'openai'; +const client = new OpenAI({ + apiKey: env.CLOUDFLARE_API_TOKEN, + baseURL: `https://api.cloudflare.com/client/v4/accounts/${env.ACCOUNT_ID}/ai/v1` +}); +``` + +## Multi-Model Setup + +```typescript +const MODELS = { + chat: '@cf/meta/llama-3.1-8b-instruct', + embed: '@cf/baai/bge-base-en-v1.5', + image: '@cf/stabilityai/stable-diffusion-xl-base-1.0' +}; +``` + +## RAG Setup (with Vectorize) + +```jsonc +{ + "ai": { "binding": "AI" }, + "vectorize": { + "bindings": [{ "binding": "VECTORIZE", "index_name": "embeddings-index" }] + } +} +``` + +## Troubleshooting + +| Error | Fix | +|-------|-----| +| `env.AI is undefined` | Check `ai` binding in wrangler.jsonc | +| Local AI doesn't work | Use `wrangler dev --remote` | +| Type 'Ai' not found | Install `@cloudflare/workers-types` | +| @cloudflare/ai package error | Don't install - use native binding | diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-ai/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workers-ai/gotchas.md new file mode 100644 index 0000000..c69255f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-ai/gotchas.md @@ -0,0 +1,114 @@ +# Workers AI Gotchas + +## Critical: @cloudflare/ai is DEPRECATED + +```typescript +// ❌ WRONG - Don't install @cloudflare/ai +import Ai from '@cloudflare/ai'; + +// ✅ CORRECT - Use native binding +export default { + async fetch(request: Request, env: Env) { + await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { messages: [...] }); + } +} +``` + +## Development + +### "AI inference doesn't work locally" +```bash +# ❌ Local AI doesn't work +wrangler dev +# ✅ Use remote +wrangler dev --remote +``` + +### "env.AI is undefined" +Add binding to wrangler.jsonc: +```jsonc +{ "ai": { "binding": "AI" } } +``` + +## API Responses + +### Embedding response shape varies +```typescript +// @cf/baai/bge-base-en-v1.5 returns: { data: [[0.1, 0.2, ...]] } +const embedding = response.data[0]; // Get first element +``` + +### Stream returns ReadableStream +```typescript +const stream = await env.AI.run(model, { messages: [...], stream: true }); +for await (const chunk of stream) { console.log(chunk.response); } +``` + +## Rate Limits & Pricing + +| Model Type | Neurons/Request | +|------------|-----------------| +| Small text (7B) | ~50-200 | +| Large text (70B) | ~500-2000 | +| Embeddings | ~5-20 | +| Image gen | ~10,000+ | + +**Free tier**: 10,000 neurons/day + +```typescript +// ❌ EXPENSIVE - 70B model +await env.AI.run('@cf/meta/llama-3.1-70b-instruct', ...); +// ✅ CHEAPER - Use smallest that works +await env.AI.run('@cf/meta/llama-3.1-8b-instruct', ...); +``` + +## Model-Specific + +### Function calling +Only `@cf/meta/llama-3.1-*` and `mistral-7b-instruct-v0.2` support tools. + +### Empty response +Check context limits (2K-8K tokens). Validate input structure. + +### Inconsistent responses +Set `temperature: 0` for deterministic outputs. + +### Cold start latency +First request: 1-3s. Use AI Gateway caching for frequent prompts. + +## TypeScript + +```typescript +interface Env { + AI: Ai; // From @cloudflare/workers-types +} + +interface TextGenerationResponse { response: string; } +interface EmbeddingResponse { data: number[][]; shape: number[]; } +``` + +## Common Errors + +### 7502: Model not found +Check exact model name at developers.cloudflare.com/workers-ai/models/ + +### 7504: Input validation failed +```typescript +// Text gen requires messages array +await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages: [{ role: 'user', content: 'Hello' }] // ✅ +}); + +// Embeddings require text +await env.AI.run('@cf/baai/bge-base-en-v1.5', { text: 'Hello' }); // ✅ +``` + +## Vercel AI SDK Integration + +```typescript +import { openai } from '@ai-sdk/openai'; +const model = openai('gpt-3.5-turbo', { + baseURL: 'https://api.cloudflare.com/client/v4/accounts//ai/v1', + headers: { Authorization: 'Bearer ' } +}); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-ai/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workers-ai/patterns.md new file mode 100644 index 0000000..8295a5b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-ai/patterns.md @@ -0,0 +1,120 @@ +# Workers AI Patterns + +## RAG (Retrieval-Augmented Generation) + +```typescript +// 1. Embed query +const embedding = await env.AI.run('@cf/baai/bge-base-en-v1.5', { text: query }); + +// 2. Search vectors +const results = await env.VECTORIZE.query(embedding.data[0], { + topK: 5, returnMetadata: true +}); + +// 3. Build context +const context = results.matches.map(m => m.metadata?.text).join('\n\n'); + +// 4. Generate with context +const response = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages: [ + { role: 'system', content: `Answer based on:\n\n${context}` }, + { role: 'user', content: query } + ] +}); +``` + +## Streaming (SSE) + +```typescript +const stream = await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { + messages, stream: true +}); + +const { readable, writable } = new TransformStream(); +const writer = writable.getWriter(); + +(async () => { + for await (const chunk of stream) { + await writer.write(new TextEncoder().encode(`data: ${JSON.stringify(chunk)}\n\n`)); + } + await writer.write(new TextEncoder().encode('data: [DONE]\n\n')); + await writer.close(); +})(); + +return new Response(readable, { + headers: { 'Content-Type': 'text/event-stream' } +}); +``` + +## Error Handling & Retry + +```typescript +async function runWithRetry(env, model, input, maxRetries = 3) { + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await env.AI.run(model, input); + } catch (error) { + if (error.message?.includes('7505') && attempt < maxRetries - 1) { + await new Promise(r => setTimeout(r, Math.pow(2, attempt) * 1000)); + continue; + } + throw error; + } + } +} +``` + +## Model Fallback + +```typescript +try { + return await env.AI.run('@cf/meta/llama-3.1-70b-instruct', { messages }); +} catch { + return await env.AI.run('@cf/meta/llama-3.1-8b-instruct', { messages }); +} +``` + +## Prompt Patterns + +```typescript +// System prompts +const PROMPTS = { + json: 'Respond with valid JSON only.', + concise: 'Keep responses brief.', + cot: 'Think step by step before answering.' +}; + +// Few-shot +messages: [ + { role: 'system', content: 'Extract as JSON' }, + { role: 'user', content: 'John bought 3 apples for $5' }, + { role: 'assistant', content: '{"name":"John","item":"apples","qty":3}' }, + { role: 'user', content: actualInput } +] +``` + +## Parallel Execution + +```typescript +const [sentiment, summary, embedding] = await Promise.all([ + env.AI.run('@cf/mistral/mistral-7b-instruct-v0.1', { messages: sentimentPrompt }), + env.AI.run('@cf/meta/llama-3.1-8b-instruct', { messages: summaryPrompt }), + env.AI.run('@cf/baai/bge-base-en-v1.5', { text }) +]); +``` + +## Cost Optimization + +| Task | Model | Neurons | +|------|-------|---------| +| Classify | `@cf/mistral/mistral-7b-instruct-v0.1` | ~50 | +| Chat | `@cf/meta/llama-3.1-8b-instruct` | ~200 | +| Complex | `@cf/meta/llama-3.1-70b-instruct` | ~2000 | +| Embed | `@cf/baai/bge-base-en-v1.5` | ~10 | + +```typescript +// Batch embeddings +const response = await env.AI.run('@cf/baai/bge-base-en-v1.5', { + text: textsArray // Process multiple at once +}); +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/README.md b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/README.md new file mode 100644 index 0000000..7bfd9b7 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/README.md @@ -0,0 +1,89 @@ +# Cloudflare Workers for Platforms + +Multi-tenant platform with isolated customer code execution at scale. + +## Use Cases + +- Multi-tenant SaaS running customer code +- AI-generated code execution in secure sandboxes +- Programmable platforms with isolated compute +- Edge functions/serverless platforms +- Website builders with static + dynamic content +- Unlimited app deployment at scale + +**NOT for general Workers** - only for Workers for Platforms architecture. + +## Quick Start + +**One-click deploy:** [Platform Starter Kit](https://github.com/cloudflare/workers-for-platforms-example) deploys complete WfP setup with dispatch namespace, dispatch worker, and user worker example. + +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https://github.com/cloudflare/workers-for-platforms-example) + +**Manual setup:** See [configuration.md](./configuration.md) for namespace creation and dispatch worker configuration. + +## Key Features + +- Unlimited Workers per namespace (no script limits) +- Automatic tenant isolation +- Custom CPU/subrequest limits per customer +- Hostname routing (subdomains/vanity domains) +- Egress/ingress control +- Static assets support +- Tags for bulk operations + +## Architecture + +**4 Components:** +1. **Dispatch Namespace** - Container for unlimited customer Workers, automatic isolation (untrusted mode by default - no request.cf access, no shared cache) +2. **Dynamic Dispatch Worker** - Entry point, routes requests, enforces platform logic (auth, limits, validation) +3. **User Workers** - Customer code in isolated sandboxes, API-deployed, optional bindings (KV/D1/R2/DO) +4. **Outbound Worker** (optional) - Intercepts external fetch, controls egress, logs subrequests (blocks TCP socket connect() API) + +**Request Flow:** +``` +Request → Dispatch Worker → Determines user Worker → env.DISPATCHER.get("customer") +→ User Worker executes (Outbound Worker for external fetch) → Response → Dispatch Worker → Client +``` + +## Decision Trees + +### When to Use Workers for Platforms +``` +Need to run code? +├─ Your code only → Regular Workers +├─ Customer/AI code → Workers for Platforms +└─ Untrusted code in sandbox → Workers for Platforms OR Sandbox API +``` + +### Routing Strategy Selection +``` +Hostname routing needed? +├─ Subdomains only (*.saas.com) → `*.saas.com/*` route + subdomain extraction +├─ Custom domains → `*/*` wildcard + Cloudflare for SaaS + KV/metadata routing +└─ Path-based (/customer/app) → Any route + path parsing +``` + +### Isolation Mode Selection +``` +Worker mode? +├─ Running customer code → Untrusted (default) +├─ Need request.cf geolocation → Trusted mode +├─ Internal platform, controlled code → Trusted mode with cache key prefixes +└─ Maximum isolation → Untrusted + unique resources per customer +``` + +## In This Reference + +| File | Purpose | When to Read | +|------|---------|--------------| +| [configuration.md](./configuration.md) | Namespace setup, dispatch worker config | First-time setup, changing limits | +| [api.md](./api.md) | User worker API, dispatch API, outbound worker | Deploying workers, SDK integration | +| [patterns.md](./patterns.md) | Multi-tenancy, routing, egress control | Planning architecture, scaling | +| [gotchas.md](./gotchas.md) | Limits, isolation issues, best practices | Debugging, production prep | + +## See Also +- [workers](../workers/) - Core Workers runtime documentation +- [durable-objects](../durable-objects/) - Stateful multi-tenant patterns +- [sandbox](../sandbox/) - Alternative for untrusted code execution +- [Reference Architecture: Programmable Platforms](https://developers.cloudflare.com/reference-architecture/diagrams/serverless/programmable-platforms/) +- [Reference Architecture: AI Vibe Coding Platform](https://developers.cloudflare.com/reference-architecture/diagrams/ai/ai-vibe-coding-platform/) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/api.md b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/api.md new file mode 100644 index 0000000..663c608 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/api.md @@ -0,0 +1,196 @@ +# API Operations + +## Deploy User Worker + +```bash +curl -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/dispatch/namespaces/$NAMESPACE/scripts/$SCRIPT_NAME" \ + -H "Authorization: Bearer $API_TOKEN" \ + -F 'metadata={"main_module": "worker.mjs"};type=application/json' \ + -F 'worker.mjs=@worker.mjs;type=application/javascript+module' +``` + +### TypeScript SDK +```typescript +import Cloudflare from "cloudflare"; + +const client = new Cloudflare({ apiToken: process.env.API_TOKEN }); + +const scriptFile = new File([scriptContent], `${scriptName}.mjs`, { + type: "application/javascript+module", +}); + +await client.workersForPlatforms.dispatch.namespaces.scripts.update( + namespace, scriptName, + { + account_id: accountId, + metadata: { main_module: `${scriptName}.mjs` }, + files: [scriptFile], + } +); +``` + +## TypeScript Types + +```typescript +import type { DispatchNamespace } from '@cloudflare/workers-types'; + +interface DispatchNamespace { + get(name: string, options?: Record, dispatchOptions?: DynamicDispatchOptions): Fetcher; +} + +interface DynamicDispatchOptions { + limits?: DynamicDispatchLimits; + outbound?: Record; +} + +interface DynamicDispatchLimits { + cpuMs?: number; // Max CPU milliseconds + subRequests?: number; // Max fetch() calls +} + +// Usage +const userWorker = env.DISPATCHER.get('customer-123', {}, { + limits: { cpuMs: 50, subRequests: 20 }, + outbound: { customerId: '123', url: request.url } +}); +``` + +## Deploy with Bindings +```bash +curl -X PUT ".../scripts/$SCRIPT_NAME" \ + -F 'metadata={ + "main_module": "worker.mjs", + "bindings": [ + {"type": "kv_namespace", "name": "MY_KV", "namespace_id": "'$KV_ID'"} + ], + "tags": ["customer-123", "production"], + "compatibility_date": "2026-01-01" // Use current date for new projects + };type=application/json' \ + -F 'worker.mjs=@worker.mjs;type=application/javascript+module' +``` + +## List/Delete Workers + +```bash +# List +curl "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/dispatch/namespaces/$NAMESPACE/scripts" \ + -H "Authorization: Bearer $API_TOKEN" + +# Delete by name +curl -X DELETE ".../scripts/$SCRIPT_NAME" -H "Authorization: Bearer $API_TOKEN" + +# Delete by tag +curl -X DELETE ".../scripts?tags=customer-123%3Ayes" -H "Authorization: Bearer $API_TOKEN" +``` + +**Pagination:** SDK supports async iteration. Manual: add `?per_page=100&page=1` query params. + +## Static Assets + +**3-step process:** Create session → Upload files → Deploy Worker + +### 1. Create Upload Session +```bash +curl -X POST ".../scripts/$SCRIPT_NAME/assets-upload-session" \ + -H "Authorization: Bearer $API_TOKEN" \ + -d '{ + "manifest": { + "/index.html": {"hash": "08f1dfda4574284ab3c21666d1ee8c7d4", "size": 1234} + } + }' +# Returns: jwt, buckets +``` + +**Hash:** SHA-256 truncated to first 16 bytes (32 hex characters) + +### 2. Upload Files +```bash +curl -X POST ".../workers/assets/upload?base64=true" \ + -H "Authorization: Bearer $UPLOAD_JWT" \ + -F '08f1dfda4574284ab3c21666d1ee8c7d4=' +# Returns: completion jwt +``` + +**Multiple buckets:** Upload to all returned bucket URLs (typically 2 for redundancy) using same JWT and hash. + +### 3. Deploy with Assets +```bash +curl -X PUT ".../scripts/$SCRIPT_NAME" \ + -F 'metadata={ + "main_module": "index.js", + "assets": {"jwt": ""}, + "bindings": [{"type": "assets", "name": "ASSETS"}] + };type=application/json' \ + -F 'index.js=export default {...};type=application/javascript+module' +``` + +**Asset Isolation:** Assets shared across namespace by default. For customer isolation, salt hash: `sha256(customerId + fileContents).slice(0, 32)` + +## Dispatch Workers + +### Subdomain Routing +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const userWorkerName = new URL(request.url).hostname.split(".")[0]; + const userWorker = env.DISPATCHER.get(userWorkerName); + return await userWorker.fetch(request); + }, +}; +``` + +### Path Routing +```typescript +const pathParts = new URL(request.url).pathname.split("/").filter(Boolean); +const userWorker = env.DISPATCHER.get(pathParts[0]); +return await userWorker.fetch(request); +``` + +### KV Routing +```typescript +const hostname = new URL(request.url).hostname; +const userWorkerName = await env.ROUTING_KV.get(hostname); +const userWorker = env.DISPATCHER.get(userWorkerName); +return await userWorker.fetch(request); +``` + +## Outbound Workers + +Control external fetch from user Workers: + +### Configure +```typescript +const userWorker = env.DISPATCHER.get( + workerName, {}, + { outbound: { customer_context: { customer_name: workerName, url: request.url } } } +); +``` + +### Implement +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const customerName = env.customer_name; + const url = new URL(request.url); + + // Block domains + if (["malicious.com"].some(d => url.hostname.includes(d))) { + return new Response("Blocked", { status: 403 }); + } + + // Inject auth + if (url.hostname === "api.example.com") { + const headers = new Headers(request.headers); + headers.set("Authorization", `Bearer ${generateJWT(customerName)}`); + return fetch(new Request(request, { headers })); + } + + return fetch(request); + }, +}; +``` + +**Note:** Doesn't intercept DO/mTLS fetch. + +See [README.md](./README.md), [configuration.md](./configuration.md), [patterns.md](./patterns.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/configuration.md new file mode 100644 index 0000000..b434999 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/configuration.md @@ -0,0 +1,167 @@ +# Configuration + +## Dispatch Namespace Binding + +### wrangler.jsonc +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "dispatch_namespaces": [{ + "binding": "DISPATCHER", + "namespace": "production" + }] +} +``` + +## Worker Isolation Mode + +Workers in a namespace run in **untrusted mode** by default for security: +- No access to `request.cf` object +- Isolated cache per Worker (no shared cache) +- `caches.default` disabled + +### Enable Trusted Mode + +For internal platforms where you control all code: + +```bash +curl -X PUT \ + "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/workers/dispatch/namespaces/$NAMESPACE" \ + -H "Authorization: Bearer $API_TOKEN" \ + -d '{"name": "'$NAMESPACE'", "trusted_workers": true}' +``` + +**Caveats:** +- Workers share cache within namespace (use cache key prefixes: `customer-${id}:${key}`) +- `request.cf` object accessible +- Redeploy existing Workers after enabling trusted mode + +**When to use:** Internal platforms, A/B testing platforms, need geolocation data + + +### With Outbound Worker +```jsonc +{ + "dispatch_namespaces": [{ + "binding": "DISPATCHER", + "namespace": "production", + "outbound": { + "service": "outbound-worker", + "parameters": ["customer_context"] + } + }] +} +``` + +## Wrangler Commands + +```bash +wrangler dispatch-namespace list +wrangler dispatch-namespace get production +wrangler dispatch-namespace create production +wrangler dispatch-namespace delete staging +wrangler dispatch-namespace rename old new +``` + +## Custom Limits + +Set CPU time and subrequest limits per invocation: + +```typescript +const userWorker = env.DISPATCHER.get( + workerName, + {}, + { + limits: { + cpuMs: 10, // Max CPU ms + subRequests: 5 // Max fetch() calls + } + } +); +``` + +Handle limit violations: +```typescript +try { + return await userWorker.fetch(request); +} catch (e) { + if (e.message.includes("CPU time limit")) { + return new Response("CPU limit exceeded", { status: 429 }); + } + throw e; +} +``` + +## Static Assets + +Deploy HTML/CSS/images with Workers. See [api.md](./api.md#static-assets) for upload process. + +### Wrangler +```jsonc +{ + "name": "customer-site", + "main": "./src/index.js", + "assets": { + "directory": "./public", + "binding": "ASSETS" + } +} +``` + +```bash +npx wrangler deploy --name customer-site --dispatch-namespace production +``` + +### Dashboard Deployment + +Alternative to CLI: + +1. Upload Worker file in dashboard +2. Add `--dispatch-namespace` flag: `wrangler deploy --dispatch-namespace production` +3. Or configure in wrangler.jsonc under `dispatch_namespaces` + +See [api.md](./api.md) for programmatic deployment via REST API or SDK. + +## Tags + +Organize/search Workers (max 8/script): + +```bash +# Set tags +curl -X PUT ".../tags" -d '["customer-123", "pro", "production"]' + +# Filter by tag +curl ".../scripts?tags=production%3Ayes" + +# Delete by tag +curl -X DELETE ".../scripts?tags=customer-123%3Ayes" +``` + +Common patterns: `customer-123`, `free|pro|enterprise`, `production|staging` + +## Bindings + +**Supported binding types:** 29 total including KV, D1, R2, Durable Objects, Analytics Engine, Service, Assets, Queue, Vectorize, Hyperdrive, Workflow, AI, Browser, and more. + +Add via API metadata (see [api.md](./api.md#deploy-with-bindings)): +```json +{ + "bindings": [ + {"type": "kv_namespace", "name": "USER_KV", "namespace_id": "..."}, + {"type": "r2_bucket", "name": "STORAGE", "bucket_name": "..."}, + {"type": "d1", "name": "DB", "id": "..."} + ] +} +``` + +Preserve existing bindings: +```json +{ + "bindings": [{"type": "r2_bucket", "name": "STORAGE", "bucket_name": "new"}], + "keep_bindings": ["kv_namespace", "d1"] // Preserves existing bindings of these types +} +``` + +For complete binding type reference, see [bindings](../bindings/) documentation + +See [README.md](./README.md), [api.md](./api.md), [patterns.md](./patterns.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/gotchas.md new file mode 100644 index 0000000..a32fe18 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/gotchas.md @@ -0,0 +1,134 @@ +# Gotchas & Limits + +## Common Errors + +### "Worker not found" + +**Cause:** Attempting to get Worker that doesn't exist in namespace +**Solution:** Catch error and return 404: + +```typescript +try { + const userWorker = env.DISPATCHER.get(workerName); + return userWorker.fetch(request); +} catch (e) { + if (e.message.startsWith("Worker not found")) { + return new Response("Worker not found", { status: 404 }); + } + throw e; // Re-throw unexpected errors +} +``` + +### "CPU time limit exceeded" + +**Cause:** User Worker exceeded configured CPU time limit +**Solution:** Track violations in Analytics Engine and return 429 response; consider adjusting limits per customer tier + +### "Hostname Routing Issues" + +**Cause:** DNS proxy settings causing routing problems +**Solution:** Use `*/*` wildcard route which works regardless of proxy settings for orange-to-orange routing + +### "Bindings Lost on Update" + +**Cause:** Not using `keep_bindings` flag when updating Worker +**Solution:** Use `keep_bindings: true` in API requests to preserve existing bindings during updates + +### "Tag Filtering Not Working" + +**Cause:** Special characters not URL encoded in tag filters +**Solution:** URL encode tags (e.g., `tags=production%3Ayes`) and avoid special chars like `,` and `&` + +### "Deploy Failures with ES Modules" + +**Cause:** Incorrect upload format for ES modules +**Solution:** Use multipart form upload, specify `main_module` in metadata, and set file type to `application/javascript+module` + +### "Static Asset Upload Failed" + +**Cause:** Invalid hash format, expired token, or incorrect encoding +**Solution:** Hash must be first 16 bytes (32 hex chars) of SHA-256, upload within 1 hour of session creation, deploy within 1 hour of upload completion, and Base64 encode file contents + +### "Outbound Worker Not Intercepting Calls" + +**Cause:** Outbound Workers don't intercept Durable Object or mTLS binding fetch +**Solution:** Plan egress control accordingly; not all fetch calls are intercepted + +### "TCP Socket Connection Failed" + +**Cause:** Outbound Worker enabled blocks `connect()` API for TCP sockets +**Solution:** Outbound Workers only intercept `fetch()` calls; TCP socket connections unavailable when outbound configured. Remove outbound if TCP needed, or use proxy pattern. + +### "API Rate Limit Exceeded" + +**Cause:** Exceeded Cloudflare API rate limits (1200 requests per 5 minutes per account, 200 requests per second per IP) +**Solution:** Implement exponential backoff: + +```typescript +async function deployWithBackoff(deploy: () => Promise, maxRetries = 3) { + for (let i = 0; i < maxRetries; i++) { + try { + return await deploy(); + } catch (e) { + if (e.status === 429 && i < maxRetries - 1) { + await new Promise(r => setTimeout(r, Math.pow(2, i) * 1000)); + continue; + } + throw e; + } + } +} +``` + +### "Gradual Deployment Not Supported" + +**Cause:** Attempted to use gradual deployments with user Workers +**Solution:** Gradual deployments not supported for Workers in dispatch namespaces. Use all-at-once deployment with staged rollout via dispatch worker logic (feature flags, percentage-based routing). + +### "Asset Session Expired" + +**Cause:** Upload JWT expired (1 hour validity) or completion token expired (1 hour after upload) +**Solution:** Complete asset upload within 1 hour of session creation, and deploy Worker within 1 hour of upload completion. For large uploads, batch files or increase upload parallelism. + +## Platform Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Workers per namespace | Unlimited | Unlike regular Workers (500 per account) | +| Namespaces per account | Unlimited | Best practice: 1 production + 1 staging | +| Max tags per Worker | 8 | For filtering and organization | +| Worker mode | Untrusted (default) | No `request.cf` access unless trusted mode | +| Cache isolation | Per-Worker (untrusted) | Shared in trusted mode with key prefixes | +| Durable Object namespaces | Unlimited | No per-account limit for WfP | +| Gradual Deployments | Not supported | All-at-once only | +| `caches.default` | Disabled (untrusted) | Use Cache API with custom keys | + +## Asset Upload Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Upload session JWT validity | 1 hour | Must complete upload within this time | +| Completion token validity | 1 hour | Must deploy within this time after upload | +| Asset hash format | First 16 bytes SHA-256 | 32 hex characters | +| Base64 encoding | Required | For binary files | + +## API Rate Limits + +| Limit Type | Value | Scope | +|------------|-------|-------| +| Client API | 1200 requests / 5 min | Per account | +| Client API | 200 requests / sec | Per IP address | +| GraphQL | Varies by query cost | Query complexity | + +See [Cloudflare API Rate Limits](https://developers.cloudflare.com/fundamentals/api/reference/limits/) for details. + +## Operational Limits + +| Operation | Limit | Notes | +|-----------|-------|-------| +| CPU time (custom limits) | Up to Workers plan limit | Set per-invocation in dispatch worker | +| Subrequests (custom limits) | Up to Workers plan limit | Set per-invocation in dispatch worker | +| Outbound Worker subrequests | Not intercepted for DO/mTLS | Only regular fetch() calls | +| TCP sockets with outbound | Disabled | `connect()` API unavailable | + +See [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/patterns.md new file mode 100644 index 0000000..d198430 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-for-platforms/patterns.md @@ -0,0 +1,188 @@ +# Multi-Tenant Patterns + +## Billing by Plan + +```typescript +interface Env { + DISPATCHER: DispatchNamespace; + CUSTOMERS_KV: KVNamespace; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const userWorkerName = new URL(request.url).hostname.split(".")[0]; + const customerPlan = await env.CUSTOMERS_KV.get(userWorkerName); + + const plans = { + enterprise: { cpuMs: 50, subRequests: 50 }, + pro: { cpuMs: 20, subRequests: 20 }, + free: { cpuMs: 10, subRequests: 5 }, + }; + const limits = plans[customerPlan as keyof typeof plans] || plans.free; + + const userWorker = env.DISPATCHER.get(userWorkerName, {}, { limits }); + return await userWorker.fetch(request); + }, +}; +``` + +## Resource Isolation + +**Complete isolation:** Create unique resources per customer +- KV namespace per customer +- D1 database per customer +- R2 bucket per customer + +```typescript +const bindings = [{ + type: "kv_namespace", + name: "USER_KV", + namespace_id: `customer-${customerId}-kv` +}]; +``` + +## Hostname Routing + +### Wildcard Route (Recommended) +Configure `*/*` route on SaaS domain → dispatch Worker + +**Benefits:** +- Supports subdomains + custom vanity domains +- No per-route limits (regular Workers limited to 100 routes) +- Programmatic control +- Works with any DNS proxy settings + +**Setup:** +1. Cloudflare for SaaS custom hostnames +2. Fallback origin (dummy `A 192.0.2.0` if Worker is origin) +3. DNS CNAME to SaaS domain +4. `*/*` route → dispatch Worker +5. Routing logic in dispatch Worker + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + const hostname = new URL(request.url).hostname; + const hostnameData = await env.ROUTING_KV.get(`hostname:${hostname}`, { type: "json" }); + + if (!hostnameData?.workerName) { + return new Response("Hostname not configured", { status: 404 }); + } + + const userWorker = env.DISPATCHER.get(hostnameData.workerName); + return await userWorker.fetch(request); + }, +}; +``` + +### Subdomain-Only +1. Wildcard DNS: `*.saas.com` → origin +2. Route: `*.saas.com/*` → dispatch Worker +3. Extract subdomain for routing + +### Orange-to-Orange (O2O) Behavior + +When customers use Cloudflare and CNAME to your Workers domain: + +| Scenario | Behavior | Route Pattern | +|----------|----------|---------------| +| Customer not on Cloudflare | Standard routing | `*/*` or `*.domain.com/*` | +| Customer on Cloudflare (proxied CNAME) | Invokes Worker at edge | `*/*` required | +| Customer on Cloudflare (DNS-only CNAME) | Standard routing | Any route works | + +**Recommendation:** Always use `*/*` wildcard for consistent O2O behavior. + +### Custom Metadata Routing + +For Cloudflare for SaaS: Store worker name in custom hostname `custom_metadata`, retrieve in dispatch worker to route requests. Requires custom hostnames as subdomains of your domain. + +## Observability + +### Logpush +- Enable on dispatch Worker → captures all user Worker logs +- Filter by `Outcome` or `Script Name` + +### Tail Workers +- Real-time logs with custom formatting +- Receives HTTP status, `console.log()`, exceptions, diagnostics + +### Analytics Engine +```typescript +// Track violations +env.ANALYTICS.writeDataPoint({ + indexes: [customerName], + blobs: ["cpu_limit_exceeded"], +}); +``` + +### GraphQL +```graphql +query { + viewer { + accounts(filter: {accountTag: $accountId}) { + workersInvocationsAdaptive(filter: {dispatchNamespaceName: "production"}) { + sum { requests errors cpuTime } + } + } + } +} +``` + +## Use Case Implementations + +### AI Code Execution +```typescript +async function deployGeneratedCode(name: string, code: string) { + const file = new File([code], `${name}.mjs`, { type: "application/javascript+module" }); + await client.workersForPlatforms.dispatch.namespaces.scripts.update("production", name, { + account_id: accountId, + metadata: { main_module: `${name}.mjs`, tags: [name, "ai-generated"] }, + files: [file], + }); +} + +// Short limits for untrusted code +const userWorker = env.DISPATCHER.get(sessionId, {}, { limits: { cpuMs: 5, subRequests: 3 } }); +``` + +**VibeSDK:** For AI-powered code generation + deployment platforms, see [VibeSDK](https://github.com/cloudflare/vibesdk) - handles AI generation, sandbox execution, live preview, and deployment. + +Reference: [AI Vibe Coding Platform Architecture](https://developers.cloudflare.com/reference-architecture/diagrams/ai/ai-vibe-coding-platform/) + +### Edge Functions Platform +```typescript +// Route: /customer-id/function-name +const [customerId, functionName] = new URL(request.url).pathname.split("/").filter(Boolean); +const workerName = `${customerId}-${functionName}`; +const userWorker = env.DISPATCHER.get(workerName); +``` + +### Website Builder +- Deploy static assets + Worker code +- See [api.md](./api.md#static-assets) for full implementation +- Salt hashes for asset isolation + +## Best Practices + +### Architecture +- One namespace per environment (production, staging) +- Platform logic in dispatch Worker (auth, rate limiting, validation) +- Isolation automatic (no shared cache, untrusted mode) + +### Routing +- Use `*/*` wildcard routes +- Store mappings in KV +- Handle missing Workers gracefully + +### Limits & Security +- Set custom limits by plan +- Track violations with Analytics Engine +- Use outbound Workers for egress control +- Sanitize responses + +### Tags +- Tag all Workers: customer ID, plan, environment +- Enable bulk operations +- Filter efficiently + +See [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-playground/README.md b/plugins/cloudflare/skills/cloudflare/references/workers-playground/README.md new file mode 100644 index 0000000..6dee4f9 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-playground/README.md @@ -0,0 +1,127 @@ +# Cloudflare Workers Playground Skill Reference + +## Overview + +Cloudflare Workers Playground is a browser-based sandbox for instantly experimenting with, testing, and deploying Cloudflare Workers without authentication or setup. This skill provides patterns, APIs, and best practices specifically for Workers Playground development. + +**URL:** [workers.cloudflare.com/playground](https://workers.cloudflare.com/playground) + +## ⚠️ Playground Constraints + +**Playground is NOT production-equivalent:** +- ✅ Real Workers runtime, instant testing, shareable URLs +- ❌ No TypeScript (JavaScript only) +- ❌ No bindings (KV, D1, R2, Durable Objects) +- ❌ No environment variables or secrets +- ❌ ES modules only (no Service Worker format) +- ⚠️ Safari broken (use Chrome/Firefox) + +**For production:** Use `wrangler` CLI. Playground is for rapid prototyping. + +## Quick Start + +Minimal Worker: + +```javascript +export default { + async fetch(request, env, ctx) { + return new Response('Hello World'); + } +}; +``` + +JSON API: + +```javascript +export default { + async fetch(request, env, ctx) { + const data = { message: 'Hello', timestamp: Date.now() }; + return Response.json(data); + } +}; +``` + +Proxy with modification: + +```javascript +export default { + async fetch(request, env, ctx) { + const response = await fetch('https://example.com'); + const modified = new Response(response.body, response); + modified.headers.set('X-Custom-Header', 'added-by-worker'); + return modified; + } +}; +``` + +Import from CDN: + +```javascript +import { Hono } from 'https://esm.sh/hono@3'; + +export default { + async fetch(request) { + const app = new Hono(); + app.get('/', (c) => c.text('Hello Hono!')); + return app.fetch(request); + } +}; +``` + +## Reading Order + +1. **[configuration.md](configuration.md)** - Start here: playground setup, constraints, deployment +2. **[api.md](api.md)** - Core APIs: Request, Response, ExecutionContext, fetch, Cache +3. **[patterns.md](patterns.md)** - Common use cases: routing, proxying, A/B testing, multi-module code +4. **[gotchas.md](gotchas.md)** - Troubleshooting: errors, browser issues, limits, best practices + +## In This Reference + +- **[configuration.md](configuration.md)** - Setup, deployment, configuration +- **[api.md](api.md)** - API endpoints, methods, interfaces +- **[patterns.md](patterns.md)** - Common patterns, use cases, examples +- **[gotchas.md](gotchas.md)** - Troubleshooting, best practices, limitations + +## Key Features + +**No Setup Required:** +- Open URL and start coding +- No CLI, no account, no config files +- Code executes in real Cloudflare Workers runtime + +**Instant Preview:** +- Live preview pane with browser tab or HTTP tester +- Auto-reload on code changes +- DevTools integration (right-click → Inspect) + +**Share & Deploy:** +- Copy Link generates permanent shareable URL +- Deploy button publishes to production in ~30 seconds +- Get `*.workers.dev` subdomain immediately + +## Common Use Cases + +- **API development:** Test endpoints before wrangler setup +- **Learning Workers:** Experiment with APIs without local environment +- **Prototyping:** Quick POCs for edge logic +- **Sharing examples:** Generate shareable links for bug reports or demos +- **Framework testing:** Import from CDN (Hono, itty-router, etc.) + +## Limitations vs Production + +| Feature | Playground | Production (wrangler) | +|---------|------------|----------------------| +| Language | JavaScript only | JS + TypeScript | +| Bindings | None | KV, D1, R2, DO, AI, etc. | +| Environment vars | None | Full support | +| Module format | ES only | ES + Service Worker | +| CPU time | 10ms (Free plan) | 10ms Free / 50ms Paid | +| Custom domains | No | Yes | +| Analytics | No | Yes | + +## See Also + +- [Cloudflare Workers Docs](https://developers.cloudflare.com/workers/) +- [Workers Examples](https://developers.cloudflare.com/workers/examples/) +- [Wrangler CLI](https://developers.cloudflare.com/workers/wrangler/) +- [Workers API Reference](https://developers.cloudflare.com/workers/runtime-apis/) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-playground/api.md b/plugins/cloudflare/skills/cloudflare/references/workers-playground/api.md new file mode 100644 index 0000000..1382ab4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-playground/api.md @@ -0,0 +1,101 @@ +# Workers Playground API + +## Handler + +```javascript +export default { + async fetch(request, env, ctx) { + // request: Request, env: {} (empty in playground), ctx: ExecutionContext + return new Response('Hello'); + } +}; +``` + +## Request + +```javascript +const method = request.method; // "GET", "POST" +const url = new URL(request.url); // Parse URL +const headers = request.headers; // Headers object +const body = await request.json(); // Read body (consumes stream) +const clone = request.clone(); // Clone before reading body + +// Query params +url.searchParams.get('page'); // Single value +url.searchParams.getAll('tag'); // Array + +// Cloudflare metadata +request.cf.country; // "US" +request.cf.colo; // "SFO" +``` + +## Response + +```javascript +// Text +return new Response('Hello', { status: 200 }); + +// JSON +return Response.json({ data }, { status: 200, headers: {...} }); + +// Redirect +return Response.redirect('/new-path', 301); + +// Modify existing +const modified = new Response(response.body, response); +modified.headers.set('X-Custom', 'value'); +``` + +## ExecutionContext + +```javascript +// Background work (after response sent) +ctx.waitUntil(fetch('https://logs.example.com', { method: 'POST', body: '...' })); +return new Response('OK'); // Returns immediately +``` + +## Fetch + +```javascript +const response = await fetch('https://api.example.com'); +const data = await response.json(); + +// With options +await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: 'Alice' }) +}); +``` + +## Cache + +```javascript +const cache = caches.default; + +// Check cache +let response = await cache.match(request); +if (!response) { + response = await fetch(origin); + await cache.put(request, response.clone()); // Clone before put! +} +return response; +``` + +## Crypto + +```javascript +crypto.randomUUID(); // UUID v4 +crypto.getRandomValues(new Uint8Array(16)); + +// SHA-256 hash +const hash = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(data)); +``` + +## Limits (Playground = Free Plan) + +| Resource | Limit | +|----------|-------| +| CPU time | 10ms | +| Subrequests | 50 | +| Memory | 128 MB | diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-playground/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workers-playground/configuration.md new file mode 100644 index 0000000..7d747ac --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-playground/configuration.md @@ -0,0 +1,163 @@ +# Configuration + +## Getting Started + +Navigate to [workers.cloudflare.com/playground](https://workers.cloudflare.com/playground) + +- **No account required** for testing +- **No CLI or local setup** needed +- Code executes in real Cloudflare Workers runtime +- Share code via URL (never expires) + +## Playground Constraints + +⚠️ **Important Limitations** + +| Constraint | Playground | Production Workers | +|------------|------------|-------------------| +| **Module Format** | ES modules only | ES modules or Service Worker | +| **TypeScript** | Not supported (JS only) | Supported via build step | +| **Bindings** | Not available | KV, D1, R2, Durable Objects, etc. | +| **wrangler.toml** | Not used | Required for config | +| **Environment Variables** | Not available | Full support | +| **Secrets** | Not available | Full support | +| **Custom Domains** | Not available | Full support | + +**Playground is for rapid prototyping only.** For production apps, use `wrangler` CLI. + +## Code Editor + +### Syntax Requirements + +Must export default object with `fetch` handler: + +```javascript +export default { + async fetch(request, env, ctx) { + return new Response('Hello World'); + } +}; +``` + +**Key Points:** +- Must use ES modules (`export default`) +- `fetch` method receives `(request, env, ctx)` +- Must return `Response` object +- TypeScript not supported (use plain JavaScript) + +### Multi-Module Code + +Import from external URLs or inline modules: + +```javascript +// Import from CDN +import { Hono } from 'https://esm.sh/hono@3'; + +// Or paste library code and import relatively +// (See patterns.md for multi-module examples) + +export default { + async fetch(request) { + const app = new Hono(); + app.get('/', (c) => c.text('Hello')); + return app.fetch(request); + } +}; +``` + +## Preview Panel + +### Browser Tab + +Default interactive preview with address bar: +- Enter custom URL paths +- Automatic reload on code changes +- DevTools available (right-click → Inspect) + +### HTTP Test Panel + +Switch to **HTTP** tab for raw HTTP testing: +- Change HTTP method (GET, POST, PUT, DELETE, PATCH, etc.) +- Add/edit request headers +- Modify request body (JSON, form data, text) +- View response headers and body +- Test different content types + +Example HTTP test: +``` +Method: POST +URL: /api/users +Headers: + Content-Type: application/json + Authorization: Bearer token123 +Body: +{ + "name": "Alice", + "email": "alice@example.com" +} +``` + +## Sharing Code + +**Copy Link** button generates shareable URL: +- Code embedded in URL fragment +- Links never expire +- No account required +- Can be bookmarked for later + +Example: `https://workers.cloudflare.com/playground#abc123...` + +## Deploying from Playground + +Click **Deploy** button to move code to production: + +1. **Log in** to Cloudflare account (creates free account if needed) +2. **Review** Worker name and code +3. **Deploy** to global network (takes ~30 seconds) +4. **Get URL**: Deployed to `.workers.dev` subdomain +5. **Manage** from dashboard: add bindings, custom domains, analytics + +**After deploy:** +- Code runs on Cloudflare's global network (300+ cities) +- Can add KV, D1, R2, Durable Objects bindings +- Configure custom domains and routes +- View analytics and logs +- Set environment variables and secrets + +**Note:** Deployed Workers are production-ready but start on Free plan (100k requests/day). + +## Browser Compatibility + +| Browser | Status | Notes | +|---------|--------|-------| +| Chrome/Edge | ✅ Full support | Recommended | +| Firefox | ✅ Full support | Works well | +| Safari | ⚠️ Broken | Preview fails with "PreviewRequestFailed" | + +**Safari users:** Use Chrome, Firefox, or Edge for Workers Playground. + +## DevTools Integration + +1. **Open preview** in browser tab +2. **Right-click** → Inspect Element +3. **Console tab** shows Worker logs: + - `console.log()` output + - Uncaught errors + - Network requests (subrequests) + +**Note:** DevTools show client-side console, not Worker execution logs. For production logging, use Logpush or Tail Workers. + +## Limits in Playground + +Same as production Free plan: + +| Resource | Limit | Notes | +|----------|-------|-------| +| CPU time | 10ms | Per request | +| Memory | 128 MB | Per request | +| Script size | 1 MB | After compression | +| Subrequests | 50 | Outbound fetch calls | +| Request size | 100 MB | Incoming | +| Response size | Unlimited | Outgoing (streamed) | + +**Exceeding CPU time** throws error immediately. Optimize hot paths or upgrade to Paid plan (50ms CPU). diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-playground/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workers-playground/gotchas.md new file mode 100644 index 0000000..9271dc4 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-playground/gotchas.md @@ -0,0 +1,88 @@ +# Workers Playground Gotchas + +## Platform Limitations + +| Limitation | Impact | Workaround | +|------------|--------|------------| +| Safari broken | Preview fails | Use Chrome/Firefox/Edge | +| TypeScript unsupported | TS syntax errors | Write plain JS or use JSDoc | +| No bindings | `env` always `{}` | Mock data or use external APIs | +| No env vars | Can't access secrets | Hardcode for testing | + +## Common Runtime Errors + +### "Response body already read" + +```javascript +// ❌ Body consumed twice +const body = await request.text(); +await fetch(url, { body: request.body }); // Error! + +// ✅ Clone first +const clone = request.clone(); +const body = await request.text(); +await fetch(url, { body: clone.body }); +``` + +### "Worker exceeded CPU time" + +**Limit:** 10ms (free), 50ms (paid) + +```javascript +// ✅ Move slow work to background +ctx.waitUntil(fetch('https://analytics.example.com', {...})); +return new Response('OK'); // Return immediately +``` + +### "Too many subrequests" + +**Limit:** 50 (free), 1000 (paid) + +```javascript +// ❌ 100 individual fetches +// ✅ Batch into single API call +await fetch('https://api.example.com/batch', { + body: JSON.stringify({ ids: [...] }) +}); +``` + +## Best Practices + +```javascript +// Clone before caching +await cache.put(request, response.clone()); +return response; + +// Validate input early +if (request.method !== 'POST') return new Response('', { status: 405 }); + +// Handle errors +try { ... } catch (e) { + return Response.json({ error: e.message }, { status: 500 }); +} +``` + +## Limits + +| Resource | Free | Paid | +|----------|------|------| +| CPU time | 10ms | 50ms | +| Memory | 128 MB | 128 MB | +| Subrequests | 50 | 1000 | + +## Browser Support + +| Browser | Status | +|---------|--------| +| Chrome | ✅ Recommended | +| Firefox | ✅ Works | +| Edge | ✅ Works | +| Safari | ❌ Broken | + +## Debugging + +```javascript +console.log('URL:', request.url); // View in browser DevTools Console +``` + +**Note:** `console.log` works in playground. For production, use Logpush or Tail Workers. diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-playground/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workers-playground/patterns.md new file mode 100644 index 0000000..4af891c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-playground/patterns.md @@ -0,0 +1,132 @@ +# Workers Playground Patterns + +## JSON API + +```javascript +export default { + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === '/api/hello') return Response.json({ message: 'Hello' }); + if (url.pathname === '/api/echo' && request.method === 'POST') { + return Response.json({ received: await request.json() }); + } + return Response.json({ error: 'Not found' }, { status: 404 }); + } +}; +``` + +## Router Pattern + +```javascript +const routes = { + '/': () => new Response('Home'), + '/api/users': () => Response.json([{ id: 1, name: 'Alice' }]) +}; + +export default { + async fetch(request) { + const handler = routes[new URL(request.url).pathname]; + return handler ? handler() : new Response('Not Found', { status: 404 }); + } +}; +``` + +## Proxy Pattern + +```javascript +export default { + async fetch(request) { + const url = new URL(request.url); + url.hostname = 'api.example.com'; + return fetch(url.toString(), { + method: request.method, headers: request.headers, body: request.body + }); + } +}; +``` + +## CORS Handling + +```javascript +export default { + async fetch(request) { + if (request.method === 'OPTIONS') { + return new Response(null, { + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization' + } + }); + } + const response = await fetch('https://api.example.com', request); + const modified = new Response(response.body, response); + modified.headers.set('Access-Control-Allow-Origin', '*'); + return modified; + } +}; +``` + +## Caching + +```javascript +export default { + async fetch(request) { + if (request.method !== 'GET') return fetch(request); + const cache = caches.default; + let response = await cache.match(request); + if (!response) { + response = await fetch('https://api.example.com'); + if (response.status === 200) await cache.put(request, response.clone()); + } + return response; + } +}; +``` + +## Hono Framework + +```javascript +import { Hono } from 'https://esm.sh/hono@3'; +const app = new Hono(); +app.get('/', (c) => c.text('Hello')); +app.get('/api/users/:id', (c) => c.json({ id: c.req.param('id') })); +app.notFound((c) => c.json({ error: 'Not found' }, 404)); +export default app; +``` + +## Authentication + +```javascript +export default { + async fetch(request) { + const auth = request.headers.get('Authorization'); + if (!auth?.startsWith('Bearer ')) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }); + } + const token = auth.substring(7); + if (token !== 'secret-token') { + return Response.json({ error: 'Invalid token' }, { status: 403 }); + } + return Response.json({ message: 'Authenticated' }); + } +}; +``` + +## Error Handling + +```javascript +export default { + async fetch(request) { + try { + const response = await fetch('https://api.example.com'); + if (!response.ok) throw new Error(`API returned ${response.status}`); + return response; + } catch (error) { + return Response.json({ error: error.message }, { status: 500 }); + } + } +}; +``` + +**Note:** In-memory state (Maps, variables) resets on Worker cold start. Use Durable Objects or KV for persistence. diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-vpc/README.md b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/README.md new file mode 100644 index 0000000..412d823 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/README.md @@ -0,0 +1,127 @@ +# Workers VPC Connectivity + +Connect Cloudflare Workers to private networks and internal infrastructure using TCP Sockets. + +## Overview + +Workers VPC connectivity enables outbound TCP connections from Workers to private resources in AWS, Azure, GCP, on-premises datacenters, or any private network. This is achieved through the **TCP Sockets API** (`cloudflare:sockets`), which provides low-level network access for custom protocols and services. + +**Key capabilities:** +- Direct TCP connections to private IPs and hostnames +- TLS/StartTLS support for encrypted connections +- Integration with Cloudflare Tunnel for secure private network access +- Full control over wire protocols (database protocols, SSH, MQTT, custom TCP) + +**Note:** This reference documents the TCP Sockets API. For the newer Workers VPC Services product (HTTP-only service bindings with built-in SSRF protection), refer to separate documentation when available. VPC Services is currently in beta (2025+). + +## Quick Decision: Which Technology? + +Need private network connectivity from Workers? + +| Requirement | Use | Why | +|------------|-----|-----| +| HTTP/HTTPS APIs in private network | VPC Services (beta, separate docs) | SSRF-safe, declarative bindings | +| PostgreSQL/MySQL databases | [Hyperdrive](../hyperdrive/) | Connection pooling, caching, optimized | +| Custom TCP protocols (SSH, MQTT, proprietary) | **TCP Sockets (this doc)** | Full protocol control | +| Simple HTTP with lowest latency | TCP Sockets + [Smart Placement](../smart-placement/) | Manual optimization | +| Expose on-prem to internet (inbound) | [Cloudflare Tunnel](../tunnel/) | Not Worker-specific | + +## When to Use TCP Sockets + +**Use TCP Sockets when you need:** +- ✅ Direct control over wire protocols (e.g., Postgres wire protocol, SSH, Redis RESP) +- ✅ Non-HTTP protocols (MQTT, SMTP, custom binary protocols) +- ✅ StartTLS or custom TLS negotiation +- ✅ Streaming binary data over TCP + +**Don't use TCP Sockets when:** +- ❌ You just need HTTP/HTTPS (use `fetch()` or VPC Services) +- ❌ You need PostgreSQL/MySQL (use Hyperdrive for pooling) +- ❌ You need WebSocket (use native Workers WebSocket) + +## Quick Start + +```typescript +import { connect } from 'cloudflare:sockets'; + +export default { + async fetch(req: Request): Promise { + // Connect to private service + const socket = connect( + { hostname: "db.internal.company.net", port: 5432 }, + { secureTransport: "on" } + ); + + try { + await socket.opened; // Wait for connection + + const writer = socket.writable.getWriter(); + await writer.write(new TextEncoder().encode("QUERY\r\n")); + await writer.close(); + + const reader = socket.readable.getReader(); + const { value } = await reader.read(); + + return new Response(value); + } finally { + await socket.close(); + } + } +}; +``` + +## Architecture Pattern: Workers + Tunnel + +Most private network connectivity combines TCP Sockets with Cloudflare Tunnel: + +``` +┌─────────┐ ┌─────────────┐ ┌──────────────┐ ┌─────────────┐ +│ Worker │────▶│ TCP Socket │────▶│ Tunnel │────▶│ Private │ +│ │ │ (this API) │ │ (cloudflared)│ │ Network │ +└─────────┘ └─────────────┘ └──────────────┘ └─────────────┘ +``` + +1. Worker opens TCP socket to Tunnel hostname +2. Tunnel endpoint routes to private IP +3. Response flows back through Tunnel to Worker + +See [configuration.md](./configuration.md) for Tunnel setup details. + +## Reading Order + +1. **Start here (README.md)** - Overview and decision guide +2. **[api.md](./api.md)** - Socket interface, types, methods +3. **[configuration.md](./configuration.md)** - Wrangler setup, Tunnel integration +4. **[patterns.md](./patterns.md)** - Real-world examples (databases, protocols, error handling) +5. **[gotchas.md](./gotchas.md)** - Limits, blocked ports, common errors + +## Key Limits + +| Limit | Value | +|-------|-------| +| Max concurrent sockets per request | 6 | +| Blocked destinations | Cloudflare IPs, localhost, port 25 | +| Scope requirement | Must create in handler (not global) | + +See [gotchas.md](./gotchas.md) for complete limits and troubleshooting. + +## Best Practices + +1. **Always close sockets** - Use try/finally blocks +2. **Validate destinations** - Prevent SSRF by allowlisting hosts +3. **Use Hyperdrive for databases** - Better performance than raw TCP +4. **Prefer fetch() for HTTP** - Only use TCP when necessary +5. **Combine with Smart Placement** - Reduce latency to private networks + +## Related Technologies + +- **[Hyperdrive](../hyperdrive/)** - PostgreSQL/MySQL with connection pooling +- **[Cloudflare Tunnel](../tunnel/)** - Secure private network access +- **[Smart Placement](../smart-placement/)** - Auto-locate Workers near backends +- **VPC Services (beta)** - HTTP-only service bindings with SSRF protection (separate docs) + +## Reference + +- [TCP Sockets API Documentation](https://developers.cloudflare.com/workers/runtime-apis/tcp-sockets/) +- [Connect to databases guide](https://developers.cloudflare.com/workers/tutorials/connect-to-postgres/) +- [Cloudflare Tunnel setup](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-vpc/api.md b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/api.md new file mode 100644 index 0000000..987fb2e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/api.md @@ -0,0 +1,202 @@ +# TCP Sockets API Reference + +Complete API reference for the Cloudflare Workers TCP Sockets API (`cloudflare:sockets`). + +## Core Function: `connect()` + +```typescript +function connect( + address: SocketAddress, + options?: SocketOptions +): Socket +``` + +Creates an outbound TCP connection to the specified address. + +### Parameters + +#### `SocketAddress` + +```typescript +interface SocketAddress { + hostname: string; // DNS hostname or IP address + port: number; // TCP port (1-65535, excluding blocked ports) +} +``` + +| Field | Type | Description | Example | +|-------|------|-------------|---------| +| `hostname` | `string` | Target hostname or IP | `"db.internal.net"`, `"10.0.1.50"` | +| `port` | `number` | TCP port number | `5432`, `443`, `22` | + +DNS names are resolved at connection time. IPv4, IPv6, and private IPs (10.x, 172.16.x, 192.168.x) supported. + +#### `SocketOptions` + +```typescript +interface SocketOptions { + secureTransport?: "off" | "on" | "starttls"; + allowHalfOpen?: boolean; +} +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `secureTransport` | `"off" \| "on" \| "starttls"` | `"off"` | TLS mode | +| `allowHalfOpen` | `boolean` | `false` | Allow half-closed connections | + +**`secureTransport` modes:** + +| Mode | Behavior | Use Case | +|------|----------|----------| +| `"off"` | Plain TCP, no encryption | Testing, internal trusted networks | +| `"on"` | Immediate TLS handshake | HTTPS, secure databases, SSH | +| `"starttls"` | Start plain, upgrade later with `startTls()` | Postgres, SMTP, IMAP | + +**`allowHalfOpen`:** When `false` (default), closing read stream auto-closes write stream. When `true`, streams are independent. + +### Returns + +A `Socket` object with readable/writable streams. + +## Socket Interface + +```typescript +interface Socket { + // Streams + readable: ReadableStream; + writable: WritableStream; + + // Connection state + opened: Promise; + closed: Promise; + + // Methods + close(): Promise; + startTls(): Socket; +} +``` + +### Properties + +#### `readable: ReadableStream` + +Stream for reading data from the socket. Use `getReader()` to consume data. + +```typescript +const reader = socket.readable.getReader(); +const { done, value } = await reader.read(); // Read one chunk +``` + +#### `writable: WritableStream` + +Stream for writing data to the socket. Use `getWriter()` to send data. + +```typescript +const writer = socket.writable.getWriter(); +await writer.write(new TextEncoder().encode("HELLO\r\n")); +await writer.close(); +``` + +#### `opened: Promise` + +Promise that resolves when connection succeeds, rejects on failure. + +```typescript +interface SocketInfo { + remoteAddress?: string; // May be undefined + localAddress?: string; // May be undefined +} + +try { + const info = await socket.opened; +} catch (error) { + // Connection failed +} +``` + +#### `closed: Promise` + +Promise that resolves when socket is fully closed (both directions). + +### Methods + +#### `close(): Promise` + +Closes the socket gracefully, waiting for pending writes to complete. + +```typescript +const socket = connect({ hostname: "api.internal", port: 443 }); +try { + // Use socket +} finally { + await socket.close(); // Always call in finally block +} +``` + +#### `startTls(): Socket` + +Upgrades connection to TLS. Only available when `secureTransport: "starttls"` was specified. + +```typescript +const socket = connect( + { hostname: "db.internal", port: 5432 }, + { secureTransport: "starttls" } +); + +// Send protocol-specific StartTLS command +const writer = socket.writable.getWriter(); +await writer.write(new TextEncoder().encode("STARTTLS\r\n")); + +// Upgrade to TLS - use returned socket, not original +const secureSocket = socket.startTls(); +const secureWriter = secureSocket.writable.getWriter(); +``` + +## Complete Example + +```typescript +import { connect } from 'cloudflare:sockets'; + +export default { + async fetch(req: Request): Promise { + const socket = connect({ hostname: "echo.example.com", port: 7 }, { secureTransport: "on" }); + + try { + await socket.opened; + + const writer = socket.writable.getWriter(); + await writer.write(new TextEncoder().encode("Hello, TCP!\n")); + await writer.close(); + + const reader = socket.readable.getReader(); + const { value } = await reader.read(); + + return new Response(value); + } finally { + await socket.close(); + } + } +}; +``` + +See [patterns.md](./patterns.md) for multi-chunk reading, error handling, and protocol implementations. + +## Quick Reference + +| Task | Code | +|------|------| +| Import | `import { connect } from 'cloudflare:sockets';` | +| Connect | `connect({ hostname: "host", port: 443 })` | +| With TLS | `connect(addr, { secureTransport: "on" })` | +| StartTLS | `socket.startTls()` after handshake | +| Write | `await writer.write(data); await writer.close();` | +| Read | `const { value } = await reader.read();` | +| Error handling | `try { await socket.opened; } catch { }` | +| Always close | `try { } finally { await socket.close(); }` | + +## See Also + +- [patterns.md](./patterns.md) - Real-world protocol implementations +- [configuration.md](./configuration.md) - Wrangler setup and environment variables +- [gotchas.md](./gotchas.md) - Limits and error handling diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-vpc/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/configuration.md new file mode 100644 index 0000000..efd2d35 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/configuration.md @@ -0,0 +1,147 @@ +# Configuration + +Setup and configuration for TCP Sockets in Cloudflare Workers. + +## Wrangler Configuration + +### Basic Setup + +TCP Sockets are available by default in Workers runtime. No special configuration required in `wrangler.jsonc`: + +```jsonc +{ + "name": "private-network-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01" +} +``` + +### Environment Variables + +Store connection details as env vars: + +```jsonc +{ + "vars": { "DB_HOST": "10.0.1.50", "DB_PORT": "5432" } +} +``` + +```typescript +interface Env { DB_HOST: string; DB_PORT: string; } + +export default { + async fetch(req: Request, env: Env): Promise { + const socket = connect({ hostname: env.DB_HOST, port: parseInt(env.DB_PORT) }); + } +}; +``` + +### Per-Environment Configuration + +```jsonc +{ + "vars": { "DB_HOST": "localhost" }, + "env": { + "staging": { "vars": { "DB_HOST": "staging-db.internal.net" } }, + "production": { "vars": { "DB_HOST": "prod-db.internal.net" } } + } +} +``` + +Deploy: `wrangler deploy --env staging` or `wrangler deploy --env production` + +## Integration with Cloudflare Tunnel + +To connect Workers to private networks, combine TCP Sockets with Cloudflare Tunnel: + +``` +Worker (TCP Socket) → Tunnel hostname → cloudflared → Private Network +``` + +### Quick Setup + +1. **Install cloudflared** on a server inside your private network +2. **Create tunnel**: `cloudflared tunnel create my-private-network` +3. **Configure routing** in `config.yml`: + +```yaml +tunnel: +credentials-file: /path/to/.json +ingress: + - hostname: db.internal.example.com + service: tcp://10.0.1.50:5432 + - service: http_status:404 # Required catch-all +``` + +4. **Run tunnel**: `cloudflared tunnel run my-private-network` +5. **Connect from Worker**: + +```typescript +const socket = connect( + { hostname: "db.internal.example.com", port: 5432 }, // Tunnel hostname + { secureTransport: "on" } +); +``` + +For detailed Tunnel setup, see [Tunnel configuration reference](../tunnel/configuration.md). + +## Smart Placement Integration + +Reduce latency by auto-placing Workers near backends: + +```jsonc +{ "placement": { "mode": "smart" } } +``` + +Workers automatically relocate closer to TCP socket destinations after observing connection latency. See [Smart Placement reference](../smart-placement/). + +## Secrets Management + +Store sensitive credentials as secrets (not in wrangler.jsonc): + +```bash +wrangler secret put DB_PASSWORD # Enter value when prompted +``` + +Access in Worker via `env.DB_PASSWORD`. Use in protocol handshake or authentication. + +## Local Development + +Test with `wrangler dev`. Note: Local mode may not access private networks. Use public endpoints or mock servers for development: + +```typescript +const config = process.env.NODE_ENV === 'dev' + ? { hostname: 'localhost', port: 5432 } // Mock + : { hostname: 'db.internal.example.com', port: 5432 }; // Production +``` + +## Connection String Patterns + +Parse connection strings to extract host and port: + +```typescript +function parseConnectionString(connStr: string): SocketAddress { + const url = new URL(connStr); // e.g., "postgres://10.0.1.50:5432/mydb" + return { hostname: url.hostname, port: parseInt(url.port) || 5432 }; +} +``` + +## Hyperdrive Integration + +For PostgreSQL/MySQL, prefer Hyperdrive over raw TCP sockets (includes connection pooling): + +```jsonc +{ "hyperdrive": [{ "binding": "DB", "id": "" }] } +``` + +See [Hyperdrive reference](../hyperdrive/) for complete setup. + +## Compatibility + +TCP Sockets available in all modern Workers. Use current date: `"compatibility_date": "2025-01-01"`. No special flags required. + +## Related Configuration + +- **[Tunnel Configuration](../tunnel/configuration.md)** - Detailed cloudflared setup +- **[Smart Placement](../smart-placement/configuration.md)** - Placement mode options +- **[Hyperdrive](../hyperdrive/configuration.md)** - Database connection pooling setup diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-vpc/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/gotchas.md new file mode 100644 index 0000000..d14faae --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/gotchas.md @@ -0,0 +1,167 @@ +# Gotchas and Troubleshooting + +Common pitfalls, limitations, and solutions for TCP Sockets in Cloudflare Workers. + +## Platform Limits + +### Connection Limits + +| Limit | Value | +|-------|-------| +| Max concurrent sockets per request | 6 (hard limit) | +| Socket lifetime | Request duration | +| Connection timeout | Platform-dependent, no setting | + +**Problem:** Exceeding 6 connections throws error + +**Solution:** Process in batches of 6 + +```typescript +for (let i = 0; i < hosts.length; i += 6) { + const batch = hosts.slice(i, i + 6).map(h => connect({ hostname: h, port: 443 })); + await Promise.all(batch.map(async s => { /* use */ await s.close(); })); +} +``` + +### Blocked Destinations + +Cloudflare IPs (1.1.1.1), localhost (127.0.0.1), port 25 (SMTP), Worker's own URL blocked for security. + +**Solution:** Use public IPs or Tunnel hostnames: `connect({ hostname: "db.internal.company.net", port: 5432 })` + +### Scope Requirements + +**Problem:** Sockets created in global scope fail + +**Cause:** Sockets tied to request lifecycle + +**Solution:** Create inside handler: `export default { async fetch() { const socket = connect(...); } }` + +## Common Errors + +### Error: "proxy request failed" + +**Causes:** Blocked destination (Cloudflare IP, localhost, port 25), DNS failure, network unreachable + +**Solution:** Validate destinations, use Tunnel hostnames, catch errors with try/catch + +### Error: "TCP Loop detected" + +**Cause:** Worker connecting to itself + +**Solution:** Connect to external service, not Worker's own hostname + +### Error: "Port 25 prohibited" + +**Cause:** SMTP port blocked + +**Solution:** Use Email Workers API for email + +### Error: "socket is not open" + +**Cause:** Read/write after close + +**Solution:** Always use try/finally to ensure proper closure order + +### Error: Connection timeout + +**Cause:** No built-in timeout + +**Solution:** Use `Promise.race()`: + +```typescript +const socket = connect(addr, opts); +const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 5000)); +await Promise.race([socket.opened, timeout]); +``` + +## TLS/SSL Issues + +### StartTLS Timing + +**Problem:** Calling `startTls()` too early + +**Solution:** Send protocol-specific STARTTLS command, wait for server OK, then call `socket.startTls()` + +### Certificate Validation + +**Problem:** Self-signed certs fail + +**Solution:** Use proper certs or Tunnel (handles TLS termination) + +## Performance Issues + +### Not Using Connection Pooling + +**Problem:** New connection overhead per request + +**Solution:** Use [Hyperdrive](../hyperdrive/) for databases (built-in pooling) + +### Not Using Smart Placement + +**Problem:** High latency to backend + +**Solution:** Enable: `{ "placement": { "mode": "smart" } }` in wrangler.jsonc + +### Forgetting to Close Sockets + +**Problem:** Resource leaks + +**Solution:** Always use try/finally: + +```typescript +const socket = connect({ hostname: "api.internal", port: 443 }); +try { + // Use socket +} finally { + await socket.close(); +} +``` + +## Data Handling Issues + +### Assuming Single Read Gets All Data + +**Problem:** Only reading once may miss chunked data + +**Solution:** Loop `reader.read()` until `done === true` (see patterns.md) + +### Text Encoding Issues + +**Problem:** Using wrong encoding + +**Solution:** Specify encoding: `new TextDecoder('iso-8859-1').decode(data)` + +## Security Issues + +### SSRF Vulnerability + +**Problem:** User-controlled destinations allow access to internal services + +**Solution:** Validate against strict allowlist: + +```typescript +const ALLOWED = ['api1.internal.net', 'api2.internal.net']; +const host = new URL(req.url).searchParams.get('host'); +if (!host || !ALLOWED.includes(host)) return new Response('Forbidden', { status: 403 }); +``` + +## When to Use Alternatives + +| Use Case | Alternative | Reason | +|----------|-------------|--------| +| PostgreSQL/MySQL | [Hyperdrive](../hyperdrive/) | Connection pooling, caching | +| HTTP/HTTPS | `fetch()` | Simpler, built-in | +| HTTP with SSRF protection | VPC Services (beta 2025+) | Declarative bindings | + +## Debugging Tips + +1. **Log connection details:** `const info = await socket.opened; console.log(info.remoteAddress);` +2. **Test with public services first:** Use tcpbin.com:4242 echo server +3. **Verify Tunnel:** `cloudflared tunnel info ` and `cloudflared tunnel route ip list` + +## Related + +- [Hyperdrive](../hyperdrive/) - Database connections +- [Smart Placement](../smart-placement/) - Latency optimization +- [Tunnel Troubleshooting](../tunnel/gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers-vpc/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/patterns.md new file mode 100644 index 0000000..392627e --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers-vpc/patterns.md @@ -0,0 +1,209 @@ +# Common Patterns + +Real-world patterns and examples for TCP Sockets in Cloudflare Workers. + +```typescript +import { connect } from 'cloudflare:sockets'; +``` + +## Basic Patterns + +### Simple Request-Response + +```typescript +const socket = connect({ hostname: "echo.example.com", port: 7 }, { secureTransport: "on" }); +try { + await socket.opened; + const writer = socket.writable.getWriter(); + await writer.write(new TextEncoder().encode("Hello\n")); + await writer.close(); + + const reader = socket.readable.getReader(); + const { value } = await reader.read(); + return new Response(value); +} finally { + await socket.close(); +} +``` + +### Reading All Data + +```typescript +async function readAll(socket: Socket): Promise { + const reader = socket.readable.getReader(); + const chunks: Uint8Array[] = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + const total = chunks.reduce((sum, c) => sum + c.length, 0); + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { result.set(chunk, offset); offset += chunk.length; } + return result; +} +``` + +### Streaming Response + +```typescript +// Stream socket data directly to HTTP response +const socket = connect({ hostname: "stream.internal", port: 9000 }, { secureTransport: "on" }); +const writer = socket.writable.getWriter(); +await writer.write(new TextEncoder().encode("STREAM\n")); +await writer.close(); +return new Response(socket.readable); +``` + +## Protocol Examples + +### Redis RESP + +```typescript +// Send: *2\r\n$3\r\nGET\r\n$\r\n\r\n +// Recv: $\r\n\r\n or $-1\r\n for null +const socket = connect({ hostname: "redis.internal", port: 6379 }); +const writer = socket.writable.getWriter(); +await writer.write(new TextEncoder().encode(`*2\r\n$3\r\nGET\r\n$3\r\nkey\r\n`)); +``` + +### PostgreSQL + +**Use [Hyperdrive](../hyperdrive/) for production.** Raw Postgres protocol is complex (startup, auth, query messages). + +### MQTT + +```typescript +const socket = connect({ hostname: "mqtt.broker", port: 1883 }); +const writer = socket.writable.getWriter(); +// CONNECT: 0x10 0x00 0x04 "MQTT" 0x04 ... +// PUBLISH: 0x30 +``` + +## Error Handling Patterns + +### Retry with Backoff + +```typescript +async function connectWithRetry(addr: SocketAddress, opts: SocketOptions, maxRetries = 3): Promise { + for (let i = 1; i <= maxRetries; i++) { + try { + const socket = connect(addr, opts); + await socket.opened; + return socket; + } catch (error) { + if (i === maxRetries) throw error; + await new Promise(r => setTimeout(r, 1000 * Math.pow(2, i - 1))); // Exponential backoff + } + } + throw new Error('Unreachable'); +} +``` + +### Timeout + +```typescript +async function connectWithTimeout(addr: SocketAddress, opts: SocketOptions, ms = 5000): Promise { + const socket = connect(addr, opts); + const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), ms)); + await Promise.race([socket.opened, timeout]); + return socket; +} +``` + +### Fallback + +```typescript +async function connectWithFallback(primary: string, fallback: string, port: number): Promise { + try { + const socket = connect({ hostname: primary, port }, { secureTransport: "on" }); + await socket.opened; + return socket; + } catch { + return connect({ hostname: fallback, port }, { secureTransport: "on" }); + } +} +``` + +## Security Patterns + +### Destination Allowlist (Prevent SSRF) + +```typescript +const ALLOWED_HOSTS = ['db.internal.company.net', 'api.internal.company.net', /^10\.0\.1\.\d+$/]; + +function isAllowed(hostname: string): boolean { + return ALLOWED_HOSTS.some(p => p instanceof RegExp ? p.test(hostname) : p === hostname); +} + +export default { + async fetch(req: Request): Promise { + const target = new URL(req.url).searchParams.get('host'); + if (!target || !isAllowed(target)) return new Response('Forbidden', { status: 403 }); + const socket = connect({ hostname: target, port: 443 }); + // Use socket... + } +}; +``` + +### Connection Pooling + +```typescript +class SocketPool { + private pool = new Map(); + + async acquire(hostname: string, port: number): Promise { + const key = `${hostname}:${port}`; + const sockets = this.pool.get(key) || []; + if (sockets.length > 0) return sockets.pop()!; + const socket = connect({ hostname, port }, { secureTransport: "on" }); + await socket.opened; + return socket; + } + + release(hostname: string, port: number, socket: Socket): void { + const key = `${hostname}:${port}`; + const sockets = this.pool.get(key) || []; + if (sockets.length < 3) { sockets.push(socket); this.pool.set(key, sockets); } + else socket.close(); + } +} +``` + +## Multi-Protocol Gateway + +```typescript +interface Protocol { name: string; defaultPort: number; test(host: string, port: number): Promise; } + +const PROTOCOLS: Record = { + redis: { + name: 'redis', + defaultPort: 6379, + async test(host, port) { + const socket = connect({ hostname: host, port }); + try { + const writer = socket.writable.getWriter(); + await writer.write(new TextEncoder().encode('*1\r\n$4\r\nPING\r\n')); + writer.releaseLock(); + const reader = socket.readable.getReader(); + const { value } = await reader.read(); + return new TextDecoder().decode(value || new Uint8Array()); + } finally { await socket.close(); } + } + } +}; + +export default { + async fetch(req: Request): Promise { + const url = new URL(req.url); + const proto = url.pathname.slice(1); // /redis + const host = url.searchParams.get('host'); + if (!host || !PROTOCOLS[proto]) return new Response('Invalid', { status: 400 }); + const result = await PROTOCOLS[proto].test(host, parseInt(url.searchParams.get('port') || '') || PROTOCOLS[proto].defaultPort); + return new Response(result); + } +}; +``` + + diff --git a/plugins/cloudflare/skills/cloudflare/references/workers/README.md b/plugins/cloudflare/skills/cloudflare/references/workers/README.md new file mode 100644 index 0000000..d5b04e1 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers/README.md @@ -0,0 +1,108 @@ +# Cloudflare Workers + +Expert guidance for building, deploying, and optimizing Cloudflare Workers applications. + +## Overview + +Cloudflare Workers run on V8 isolates (NOT containers/VMs): +- Extremely fast cold starts (< 1ms) +- Global deployment across 300+ locations +- Web standards compliant (fetch, URL, Headers, Request, Response) +- Support JS/TS, Python, Rust, and WebAssembly + +**Key principle**: Workers use web platform APIs wherever possible for portability. + +## Module Worker Pattern (Recommended) + +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + return new Response('Hello World!'); + }, +}; +``` + +**Handler parameters**: +- `request`: Incoming HTTP request (standard Request object) +- `env`: Environment bindings (KV, D1, R2, secrets, vars) +- `ctx`: Execution context (`waitUntil`, `passThroughOnException`) + +## Essential Commands + +```bash +npx wrangler dev # Local dev +npx wrangler dev --remote # Remote dev (actual resources) +npx wrangler deploy # Production +npx wrangler deploy --env staging # Specific environment +npx wrangler tail # Stream logs +npx wrangler secret put API_KEY # Set secret +``` + +## When to Use Workers + +- API endpoints at the edge +- Request/response transformation +- Authentication/authorization layers +- Static asset optimization +- A/B testing and feature flags +- Rate limiting and security +- Proxy/routing logic +- WebSocket applications + +## Quick Start + +```bash +npm create cloudflare@latest my-worker -- --type hello-world +cd my-worker +npx wrangler dev +``` + +## Handler Signatures + +```typescript +// HTTP requests +async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise + +// Cron triggers +async scheduled(event: ScheduledEvent, env: Env, ctx: ExecutionContext): Promise + +// Queue consumer +async queue(batch: MessageBatch, env: Env, ctx: ExecutionContext): Promise + +// Tail consumer +async tail(events: TraceItem[], env: Env, ctx: ExecutionContext): Promise +``` + +## Resources + +**Docs**: https://developers.cloudflare.com/workers/ +**Examples**: https://developers.cloudflare.com/workers/examples/ +**Runtime APIs**: https://developers.cloudflare.com/workers/runtime-apis/ + +## In This Reference + +- [Configuration](./configuration.md) - wrangler.jsonc setup, bindings, environments +- [API](./api.md) - Runtime APIs, bindings, execution context +- [Patterns](./patterns.md) - Common workflows, testing, optimization +- [Frameworks](./frameworks.md) - Hono, routing, validation +- [Gotchas](./gotchas.md) - Common issues, limits, troubleshooting + +## Reading Order + +| Task | Start With | Then Read | +|------|------------|-----------| +| First Worker | README → Configuration → API | Patterns | +| Add framework | Frameworks | Configuration (bindings) | +| Add storage/bindings | Configuration → API (binding usage) | See Also links | +| Debug issues | Gotchas | API (specific binding docs) | +| Production optimization | Patterns | API (caching, streaming) | +| Type safety | Configuration (TypeScript) | Frameworks (Hono typing) | + +## See Also + +- [KV](../kv/README.md) - Key-value storage +- [D1](../d1/README.md) - SQL database +- [R2](../r2/README.md) - Object storage +- [Durable Objects](../durable-objects/README.md) - Stateful coordination +- [Queues](../queues/README.md) - Message queues +- [Wrangler](../wrangler/README.md) - CLI tool reference diff --git a/plugins/cloudflare/skills/cloudflare/references/workers/api.md b/plugins/cloudflare/skills/cloudflare/references/workers/api.md new file mode 100644 index 0000000..a4fc13f --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers/api.md @@ -0,0 +1,195 @@ +# Workers Runtime APIs + +## Fetch Handler + +```typescript +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + const url = new URL(request.url); + if (request.method === 'POST' && url.pathname === '/api') { + const body = await request.json(); + return new Response(JSON.stringify({ id: 1 }), { + headers: { 'Content-Type': 'application/json' } + }); + } + return fetch(request); // Subrequest to origin + }, +}; +``` + +## Execution Context + +```typescript +ctx.waitUntil(logAnalytics(request)); // Background work, don't block response +ctx.passThroughOnException(); // Failover to origin on error +``` + +**Never** `await` background operations - use `ctx.waitUntil()`. + +## Bindings + +```typescript +// KV +await env.MY_KV.get('key'); +await env.MY_KV.put('key', 'value', { expirationTtl: 3600 }); + +// R2 +const obj = await env.MY_BUCKET.get('file.txt'); +await env.MY_BUCKET.put('file.txt', 'content'); + +// D1 +const result = await env.DB.prepare('SELECT * FROM users WHERE id = ?').bind(1).first(); + +// D1 Sessions (2024+) - read-after-write consistency +const session = env.DB.withSession(); +await session.prepare('INSERT INTO users (name) VALUES (?)').bind('Alice').run(); +const user = await session.prepare('SELECT * FROM users WHERE name = ?').bind('Alice').first(); // Guaranteed fresh + +// Queues +await env.MY_QUEUE.send({ timestamp: Date.now() }); + +// Secrets/vars +const key = env.API_KEY; +``` + +## Cache API + +```typescript +const cache = caches.default; +let response = await cache.match(request); + +if (!response) { + response = await fetch(request); + response = new Response(response.body, response); + response.headers.set('Cache-Control', 'max-age=3600'); + ctx.waitUntil(cache.put(request, response.clone())); // Clone before caching +} +``` + +## HTMLRewriter + +```typescript +return new HTMLRewriter() + .on('a[href]', { + element(el) { + const href = el.getAttribute('href'); + if (href?.startsWith('http://')) { + el.setAttribute('href', href.replace('http://', 'https://')); + } + } + }) + .transform(response); +``` + +**Use cases**: A/B testing, analytics injection, link rewriting + +## WebSockets + +### Standard WebSocket + +```typescript +const [client, server] = Object.values(new WebSocketPair()); + +server.accept(); +server.addEventListener('message', event => { + server.send(`Echo: ${event.data}`); +}); + +return new Response(null, { status: 101, webSocket: client }); +``` + +### WebSocket Hibernation (Recommended for idle connections) + +```typescript +// In Durable Object +export class WebSocketDO { + async webSocketMessage(ws: WebSocket, message: string) { + ws.send(`Echo: ${message}`); + } + + async webSocketClose(ws: WebSocket, code: number, reason: string) { + // Cleanup on close + } + + async webSocketError(ws: WebSocket, error: Error) { + console.error('WebSocket error:', error); + } +} +``` + +Hibernation automatically suspends inactive connections (no CPU cost), wakes on events + +## Durable Objects + +### RPC Pattern (Recommended 2024+) + +```typescript +export class Counter { + private value = 0; + + constructor(private state: DurableObjectState) { + state.blockConcurrencyWhile(async () => { + this.value = (await state.storage.get('value')) || 0; + }); + } + + // Export methods directly - called via RPC (type-safe, zero serialization) + async increment(): Promise { + this.value++; + await this.state.storage.put('value', this.value); + return this.value; + } + + async getValue(): Promise { + return this.value; + } +} + +// Worker usage: +const stub = env.COUNTER.get(env.COUNTER.idFromName('global')); +const count = await stub.increment(); // Direct method call, full type safety +``` + +### Legacy Fetch Pattern (Pre-2024) + +```typescript +async fetch(request: Request): Promise { + const url = new URL(request.url); + if (url.pathname === '/increment') { + await this.state.storage.put('value', ++this.value); + } + return new Response(String(this.value)); +} +// Usage: await stub.fetch('http://x/increment') +``` + +**When to use DOs**: Real-time collaboration, rate limiting, strongly consistent state + +## Other Handlers + +```typescript +// Cron: async scheduled(event, env, ctx) { ctx.waitUntil(doCleanup(env)); } +// Queue: async queue(batch) { for (const msg of batch.messages) { await process(msg.body); msg.ack(); } } +// Tail: async tail(events, env) { for (const e of events) if (e.outcome === 'exception') await log(e); } +``` + +## Service Bindings + +```typescript +// Worker-to-worker RPC (zero latency, no internet round-trip) +return env.SERVICE_B.fetch(request); + +// With RPC (2024+) - same as Durable Objects RPC +export class ServiceWorker { + async getData() { return { data: 'value' }; } +} +// Usage: const data = await env.SERVICE_B.getData(); +``` + +**Benefits**: Type-safe method calls, no HTTP overhead, share code between Workers + +## See Also + +- [Configuration](./configuration.md) - Binding setup +- [Patterns](./patterns.md) - Common workflows +- [KV](../kv/README.md), [D1](../d1/README.md), [R2](../r2/README.md), [Durable Objects](../durable-objects/README.md), [Queues](../queues/README.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workers/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workers/configuration.md new file mode 100644 index 0000000..9eae70b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers/configuration.md @@ -0,0 +1,185 @@ +# Workers Configuration + +## wrangler.jsonc (Recommended) + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + + // Bindings (non-inheritable) + "vars": { "ENVIRONMENT": "production" }, + "kv_namespaces": [{ "binding": "MY_KV", "id": "abc123" }], + "r2_buckets": [{ "binding": "MY_BUCKET", "bucket_name": "my-bucket" }], + "d1_databases": [{ "binding": "DB", "database_name": "my-db", "database_id": "xyz789" }], + + // Environments + "env": { + "staging": { + "vars": { "ENVIRONMENT": "staging" }, + "kv_namespaces": [{ "binding": "MY_KV", "id": "staging-id" }] + } + } +} +``` + +## Configuration Rules + +**Inheritable**: `name`, `main`, `compatibility_date`, `routes`, `workers_dev` +**Non-inheritable**: All bindings (`vars`, `kv_namespaces`, `r2_buckets`, etc.) +**Top-level only**: `migrations`, `keep_vars`, `send_metrics` + +**ALWAYS set `compatibility_date` to current date for new projects** + +## Bindings + +```jsonc +{ + // Environment variables - access via env.VAR_NAME + "vars": { "ENVIRONMENT": "production" }, + + // KV (key-value storage) + "kv_namespaces": [{ "binding": "MY_KV", "id": "abc123" }], + + // R2 (object storage) + "r2_buckets": [{ "binding": "MY_BUCKET", "bucket_name": "my-bucket" }], + + // D1 (SQL database) + "d1_databases": [{ "binding": "DB", "database_name": "my-db", "database_id": "xyz789" }], + + // Durable Objects (stateful coordination) + "durable_objects": { + "bindings": [{ "name": "COUNTER", "class_name": "Counter" }] + }, + + // Queues (message queues) + "queues": { + "producers": [{ "binding": "MY_QUEUE", "queue": "my-queue" }], + "consumers": [{ "queue": "my-queue", "max_batch_size": 10 }] + }, + + // Service bindings (worker-to-worker RPC) + "services": [{ "binding": "SERVICE_B", "service": "service-b" }], + + // Analytics Engine + "analytics_engine_datasets": [{ "binding": "ANALYTICS" }] +} +``` + +### Secrets + +Set via CLI (never in config): + +```bash +npx wrangler secret put API_KEY +``` + +Access: `env.API_KEY` + +### Automatic Provisioning (Beta) + +Bindings without IDs are auto-created: + +```jsonc +{ "kv_namespaces": [{ "binding": "MY_KV" }] } // ID added on deploy +``` + +## Routes & Triggers + +```jsonc +{ + "routes": [ + { "pattern": "example.com/*", "zone_name": "example.com" } + ], + "triggers": { + "crons": ["0 */6 * * *"] // Every 6 hours + } +} +``` + +## TypeScript Setup + +### Automatic Type Generation (Recommended) + +```bash +npm install -D @cloudflare/workers-types +npx wrangler types # Generates .wrangler/types/runtime.d.ts from wrangler.jsonc +``` + +`tsconfig.json`: + +```jsonc +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "types": ["@cloudflare/workers-types"] + }, + "include": [".wrangler/types/**/*.ts", "src/**/*"] +} +``` + +Import generated types: + +```typescript +import type { Env } from './.wrangler/types/runtime'; + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + await env.MY_KV.get('key'); // Fully typed, autocomplete works + return new Response('OK'); + }, +}; +``` + +Re-run `npx wrangler types` after changing bindings in wrangler.jsonc + +### Manual Type Definition (Legacy) + +```typescript +interface Env { + MY_KV: KVNamespace; + DB: D1Database; + API_KEY: string; +} +``` + +## Advanced Options + +```jsonc +{ + // Auto-locate compute near data sources + "placement": { "mode": "smart" }, + + // Enable Node.js built-ins (Buffer, process, path, etc.) + "compatibility_flags": ["nodejs_compat_v2"], + + // Observability (10% sampling) + "observability": { "enabled": true, "head_sampling_rate": 0.1 } +} +``` + +### Node.js Compatibility + +`nodejs_compat_v2` enables: +- `Buffer`, `process.env`, `path`, `stream` +- CommonJS `require()` for Node modules +- `node:` imports (e.g., `import { Buffer } from 'node:buffer'`) + +**Note:** Adds ~1-2ms cold start overhead. Use Workers APIs (R2, KV) when possible + +## Deployment Commands + +```bash +npx wrangler deploy # Production +npx wrangler deploy --env staging +npx wrangler deploy --dry-run # Validate only +``` + +## See Also + +- [API](./api.md) - Runtime APIs and bindings usage +- [Patterns](./patterns.md) - Deployment strategies +- [Wrangler](../wrangler/README.md) - CLI reference diff --git a/plugins/cloudflare/skills/cloudflare/references/workers/frameworks.md b/plugins/cloudflare/skills/cloudflare/references/workers/frameworks.md new file mode 100644 index 0000000..6089e6d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers/frameworks.md @@ -0,0 +1,197 @@ +# Workers Frameworks + +## Hono (Recommended) + +Workers-native web framework with excellent TypeScript support and middleware ecosystem. + +```bash +npm install hono +``` + +### Basic Setup + +```typescript +import { Hono } from 'hono'; + +const app = new Hono(); + +app.get('/', (c) => c.text('Hello World!')); +app.post('/api/users', async (c) => { + const body = await c.req.json(); + return c.json({ id: 1, ...body }, 201); +}); + +export default app; +``` + +### Typed Environment + +```typescript +import type { Env } from './.wrangler/types/runtime'; + +const app = new Hono<{ Bindings: Env }>(); + +app.get('/data', async (c) => { + const value = await c.env.MY_KV.get('key'); // Fully typed + return c.text(value || 'Not found'); +}); +``` + +### Middleware + +```typescript +import { cors } from 'hono/cors'; +import { logger } from 'hono/logger'; + +app.use('*', logger()); +app.use('/api/*', cors({ origin: '*' })); + +// Custom middleware +app.use('/protected/*', async (c, next) => { + const auth = c.req.header('Authorization'); + if (!auth?.startsWith('Bearer ')) return c.text('Unauthorized', 401); + await next(); +}); +``` + +### Request Validation (Zod) + +```typescript +import { zValidator } from '@hono/zod-validator'; +import { z } from 'zod'; + +const schema = z.object({ + name: z.string().min(1), + email: z.string().email(), +}); + +app.post('/users', zValidator('json', schema), async (c) => { + const validated = c.req.valid('json'); // Type-safe, validated data + return c.json({ id: 1, ...validated }); +}); +``` + +**Error handling**: Automatic 400 response with validation errors + +### Route Groups + +```typescript +const api = new Hono().basePath('/api'); + +api.get('/users', (c) => c.json([])); +api.post('/users', (c) => c.json({ id: 1 })); + +app.route('/', api); // Mounts at /api/* +``` + +### Error Handling + +```typescript +app.onError((err, c) => { + console.error(err); + return c.json({ error: err.message }, 500); +}); + +app.notFound((c) => c.json({ error: 'Not Found' }, 404)); +``` + +### Accessing ExecutionContext + +```typescript +export default { + fetch(request: Request, env: Env, ctx: ExecutionContext) { + return app.fetch(request, env, ctx); + }, +}; + +// In route handlers: +app.get('/log', (c) => { + c.executionCtx.waitUntil(logRequest(c.req)); + return c.text('OK'); +}); +``` + +### OpenAPI/Swagger (Hono OpenAPI) + +```typescript +import { OpenAPIHono, createRoute, z } from '@hono/zod-openapi'; + +const app = new OpenAPIHono(); + +const route = createRoute({ + method: 'get', + path: '/users/{id}', + request: { params: z.object({ id: z.string() }) }, + responses: { + 200: { description: 'User found', content: { 'application/json': { schema: z.object({ id: z.string() }) } } }, + }, +}); + +app.openapi(route, (c) => { + const { id } = c.req.valid('param'); + return c.json({ id }); +}); + +app.doc('/openapi.json', { openapi: '3.0.0', info: { version: '1.0.0', title: 'API' } }); +``` + +### Testing with Hono + +```typescript +import { describe, it, expect } from 'vitest'; +import app from '../src/index'; + +describe('API', () => { + it('GET /', async () => { + const res = await app.request('/'); + expect(res.status).toBe(200); + expect(await res.text()).toBe('Hello World!'); + }); +}); +``` + +## Other Frameworks + +### itty-router (Minimalist) + +```typescript +import { Router } from 'itty-router'; + +const router = Router(); + +router.get('/users/:id', ({ params }) => new Response(params.id)); + +export default { fetch: router.handle }; +``` + +**Use case**: Tiny bundle size (~500 bytes), simple routing needs + +### Worktop (Advanced) + +```typescript +import { Router } from 'worktop'; + +const router = new Router(); + +router.add('GET', '/users/:id', (req, res) => { + res.send(200, { id: req.params.id }); +}); + +router.listen(); +``` + +**Use case**: Advanced routing, built-in CORS/cache utilities + +## Framework Comparison + +| Framework | Bundle Size | TypeScript | Middleware | Validation | Best For | +|-----------|-------------|------------|------------|------------|----------| +| Hono | ~12KB | Excellent | Rich | Zod | Production apps | +| itty-router | ~500B | Good | Basic | Manual | Minimal APIs | +| Worktop | ~8KB | Good | Advanced | Manual | Complex routing | + +## See Also + +- [Patterns](./patterns.md) - Common workflows +- [API](./api.md) - Runtime APIs +- [Gotchas](./gotchas.md) - Framework-specific issues diff --git a/plugins/cloudflare/skills/cloudflare/references/workers/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workers/gotchas.md new file mode 100644 index 0000000..d01811b --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers/gotchas.md @@ -0,0 +1,136 @@ +# Workers Gotchas + +## Common Errors + +### "Too much CPU time used" + +**Cause:** Worker exceeded CPU time limit (10ms standard, 30ms unbound) +**Solution:** Use `ctx.waitUntil()` for background work, offload heavy compute to Durable Objects, or consider Workers AI for ML workloads + +### "Module-Level State Lost" + +**Cause:** Workers are stateless between requests; module-level variables reset unpredictably +**Solution:** Use KV, D1, or Durable Objects for persistent state; don't rely on module-level variables + +### "Body has already been used" + +**Cause:** Attempting to read response body twice (bodies are streams) +**Solution:** Clone response before reading: `response.clone()` or read once and create new Response with the text + +### "Node.js module not found" + +**Cause:** Node.js built-ins not available by default +**Solution:** Use Workers APIs (e.g., R2 for file storage) or enable Node.js compat with `"compatibility_flags": ["nodejs_compat_v2"]` + +### "Cannot fetch in global scope" + +**Cause:** Attempting to use fetch during module initialization +**Solution:** Move fetch calls inside handler functions (fetch, scheduled, etc.) where they're allowed + +### "Subrequest depth limit exceeded" + +**Cause:** Too many nested subrequests creating deep call chain +**Solution:** Flatten request chain or use service bindings for direct Worker-to-Worker communication + +### "D1 read-after-write inconsistency" + +**Cause:** D1 is eventually consistent; reads may not reflect recent writes +**Solution:** Use D1 Sessions (2024+) to guarantee read-after-write consistency within a session: + +```typescript +const session = env.DB.withSession(); +await session.prepare('INSERT INTO users (name) VALUES (?)').bind('Alice').run(); +const user = await session.prepare('SELECT * FROM users WHERE name = ?').bind('Alice').first(); // Guaranteed to see Alice +``` + +**When to use sessions:** Write → Read patterns, transactions requiring consistency + +### "wrangler types not generating TypeScript definitions" + +**Cause:** Type generation not configured or outdated +**Solution:** Run `npx wrangler types` after changing bindings in wrangler.jsonc: + +```bash +npx wrangler types # Generates .wrangler/types/runtime.d.ts +``` + +Add to `tsconfig.json`: `"include": [".wrangler/types/**/*.ts"]` + +Then import: `import type { Env } from './.wrangler/types/runtime';` + +### "Durable Object RPC errors with deprecated fetch pattern" + +**Cause:** Using old `stub.fetch()` pattern instead of RPC (2024+) +**Solution:** Export methods directly, call via RPC: + +```typescript +// ❌ Old fetch pattern +export class MyDO { + async fetch(request: Request) { + const { method } = await request.json(); + if (method === 'increment') return new Response(String(await this.increment())); + } + async increment() { return ++this.value; } +} +const stub = env.DO.get(id); +const res = await stub.fetch('http://x', { method: 'POST', body: JSON.stringify({ method: 'increment' }) }); + +// ✅ RPC pattern (type-safe, no serialization overhead) +export class MyDO { + async increment() { return ++this.value; } +} +const stub = env.DO.get(id); +const count = await stub.increment(); // Direct method call +``` + +### "WebSocket connection closes unexpectedly" + +**Cause:** Worker reaches CPU limit while maintaining WebSocket connection +**Solution:** Use WebSocket hibernation (2024+) to offload idle connections: + +```typescript +export class WebSocketDO { + async webSocketMessage(ws: WebSocket, message: string) { + // Handle message + } + async webSocketClose(ws: WebSocket, code: number) { + // Cleanup + } +} +``` + +Hibernation automatically suspends inactive connections, wakes on events + +### "Framework middleware not working with Workers" + +**Cause:** Framework expects Node.js primitives (e.g., Express uses Node streams) +**Solution:** Use Workers-native frameworks (Hono, itty-router, Worktop) or adapt middleware: + +```typescript +// ✅ Hono (Workers-native) +import { Hono } from 'hono'; +const app = new Hono(); +app.use('*', async (c, next) => { /* middleware */ await next(); }); +``` + +See [frameworks.md](./frameworks.md) for full patterns + +## Limits + +| Limit | Value | Notes | +|-------|-------|-------| +| Request size | 100 MB | Maximum incoming request size | +| Response size | Unlimited | Supports streaming | +| CPU time (standard) | 10ms | Standard Workers | +| CPU time (unbound) | 30ms | Unbound Workers | +| Subrequests | 1000 | Per request | +| KV reads | 1000 | Per request | +| KV write size | 25 MB | Maximum per write | +| Environment size | 5 MB | Total size of env bindings | + +## See Also + +- [Patterns](./patterns.md) - Best practices +- [API](./api.md) - Runtime APIs +- [Configuration](./configuration.md) - Setup +- [Frameworks](./frameworks.md) - Hono, routing, validation diff --git a/plugins/cloudflare/skills/cloudflare/references/workers/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workers/patterns.md new file mode 100644 index 0000000..7768c4d --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workers/patterns.md @@ -0,0 +1,198 @@ +# Workers Patterns + +## Error Handling + +```typescript +class HTTPError extends Error { + constructor(public status: number, message: string) { super(message); } +} + +export default { + async fetch(request: Request, env: Env): Promise { + try { + return await handleRequest(request, env); + } catch (error) { + if (error instanceof HTTPError) { + return new Response(JSON.stringify({ error: error.message }), { + status: error.status, headers: { 'Content-Type': 'application/json' } + }); + } + return new Response('Internal Server Error', { status: 500 }); + } + }, +}; +``` + +## CORS + +```typescript +const corsHeaders = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS' }; +if (request.method === 'OPTIONS') return new Response(null, { headers: corsHeaders }); +``` + +## Routing + +```typescript +const router = { 'GET /api/users': handleGetUsers, 'POST /api/users': handleCreateUser }; + +const handler = router[`${request.method} ${url.pathname}`]; +return handler ? handler(request, env) : new Response('Not Found', { status: 404 }); +``` + +**Production**: Use Hono, itty-router, or Worktop (see [frameworks.md](./frameworks.md)) + +## Request Validation (Zod) + +```typescript +import { z } from 'zod'; + +const userSchema = z.object({ + name: z.string().min(1).max(100), + email: z.string().email(), + age: z.number().int().positive().optional(), +}); + +async function handleCreateUser(request: Request) { + try { + const body = await request.json(); + const validated = userSchema.parse(body); // Throws on invalid data + return new Response(JSON.stringify({ id: 1, ...validated }), { + status: 201, + headers: { 'Content-Type': 'application/json' }, + }); + } catch (err) { + if (err instanceof z.ZodError) { + return new Response(JSON.stringify({ errors: err.errors }), { status: 400 }); + } + throw err; + } +} +``` + +**With Hono**: Use `@hono/zod-validator` for automatic validation (see [frameworks.md](./frameworks.md)) + +## Performance + +```typescript +// ❌ Sequential +const user = await fetch('/api/user/1'); +const posts = await fetch('/api/posts?user=1'); + +// ✅ Parallel +const [user, posts] = await Promise.all([fetch('/api/user/1'), fetch('/api/posts?user=1')]); +``` + +## Streaming + +```typescript +const stream = new ReadableStream({ + async start(controller) { + for (let i = 0; i < 1000; i++) { + controller.enqueue(new TextEncoder().encode(`Item ${i}\n`)); + if (i % 100 === 0) await new Promise(r => setTimeout(r, 0)); + } + controller.close(); + } +}); +``` + +## Transform Streams + +```typescript +response.body.pipeThrough(new TextDecoderStream()).pipeThrough( + new TransformStream({ transform(chunk, c) { c.enqueue(chunk.toUpperCase()); } }) +).pipeThrough(new TextEncoderStream()); +``` + +## Testing + +```typescript +import { describe, it, expect } from 'vitest'; +import worker from '../src/index'; + +describe('Worker', () => { + it('returns 200', async () => { + const req = new Request('http://localhost/'); + const env = { MY_VAR: 'test' }; + const ctx = { waitUntil: () => {}, passThroughOnException: () => {} }; + expect((await worker.fetch(req, env, ctx)).status).toBe(200); + }); +}); +``` + +## Deployment + +```bash +npx wrangler deploy # production +npx wrangler deploy --env staging +npx wrangler versions upload --message "Add feature" +npx wrangler rollback +``` + +## Monitoring + +```typescript +const start = Date.now(); +const response = await handleRequest(request, env); +ctx.waitUntil(env.ANALYTICS.writeDataPoint({ + doubles: [Date.now() - start], blobs: [request.url, String(response.status)] +})); +``` + +## Security & Rate Limiting + +```typescript +// Security headers +const security = { 'X-Content-Type-Options': 'nosniff', 'X-Frame-Options': 'DENY' }; + +// Auth +const auth = request.headers.get('Authorization'); +if (!auth?.startsWith('Bearer ')) return new Response('Unauthorized', { status: 401 }); + +// Gradual rollouts (deterministic user bucketing) +const hash = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(userId)); +if (new Uint8Array(hash)[0] % 100 < rolloutPercent) return newFeature(request); +``` + +Rate limiting: See [Durable Objects](../durable-objects/README.md) + +## R2 Multipart Upload + +```typescript +// For files > 100MB +const upload = await env.MY_BUCKET.createMultipartUpload('large-file.bin'); +try { + const parts = []; + for (let i = 0; i < chunks.length; i++) { + parts.push(await upload.uploadPart(i + 1, chunks[i])); + } + await upload.complete(parts); +} catch (err) { await upload.abort(); throw err; } +``` + +Parallel uploads, resume on failure, handle files > 5GB + +## Workflows (Step Orchestration) + +```typescript +import { WorkflowEntrypoint, WorkflowStep, WorkflowEvent } from 'cloudflare:workers'; + +export class MyWorkflow extends WorkflowEntrypoint { + async run(event: WorkflowEvent<{ userId: string }>, step: WorkflowStep) { + const user = await step.do('fetch-user', async () => + fetch(`/api/users/${event.payload.userId}`).then(r => r.json()) + ); + await step.sleep('wait', '1 hour'); + await step.do('notify', async () => sendEmail(user.email)); + } +} +``` + +Multi-step jobs with automatic retries, state persistence, resume from failure + +## See Also + +- [API](./api.md) - Runtime APIs +- [Gotchas](./gotchas.md) - Common issues +- [Configuration](./configuration.md) - Setup +- [Frameworks](./frameworks.md) - Hono, routing, validation diff --git a/plugins/cloudflare/skills/cloudflare/references/workflows/README.md b/plugins/cloudflare/skills/cloudflare/references/workflows/README.md new file mode 100644 index 0000000..561f907 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workflows/README.md @@ -0,0 +1,69 @@ +# Cloudflare Workflows + +Durable multi-step applications with automatic retries, state persistence, and long-running execution. + +## What It Does + +- Chain steps with automatic retry logic +- Persist state between steps (minutes → weeks) +- Handle failures without losing progress +- Wait for external events/approvals +- Sleep without consuming resources + +**Available:** Free & Paid Workers plans + +## Core Concepts + +**Workflow**: Class extending `WorkflowEntrypoint` with `run` method +**Instance**: Single execution with unique ID & independent state +**Steps**: Independently retriable units via `step.do()` - API calls, DB queries, AI invocations +**State**: Persisted from step returns; step name = cache key + +## Quick Start + +```typescript +import { WorkflowEntrypoint, WorkflowStep, WorkflowEvent } from 'cloudflare:workers'; + +type Env = { MY_WORKFLOW: Workflow; DB: D1Database }; +type Params = { userId: string }; + +export class MyWorkflow extends WorkflowEntrypoint { + async run(event: WorkflowEvent, step: WorkflowStep) { + const user = await step.do('fetch user', async () => { + return await this.env.DB.prepare('SELECT * FROM users WHERE id = ?') + .bind(event.params.userId).first(); + }); + + await step.sleep('wait 7 days', '7 days'); + + await step.do('send reminder', async () => { + await sendEmail(user.email, 'Reminder!'); + }); + } +} +``` + +## Key Features + +- **Durability**: Failed steps don't re-run successful ones +- **Retries**: Configurable backoff (constant/linear/exponential) +- **Events**: `waitForEvent()` for webhooks/approvals (timeout: 1h → 365d) +- **Sleep**: `sleep()` / `sleepUntil()` for scheduling (max 365d) +- **Parallel**: `Promise.all()` for concurrent steps +- **Idempotency**: Check-then-execute patterns + +## Reading Order + +**Getting Started:** configuration.md → api.md → patterns.md +**Troubleshooting:** gotchas.md + +## In This Reference +- [configuration.md](./configuration.md) - wrangler.jsonc setup, step config, bindings +- [api.md](./api.md) - Step APIs, instance management, sleep/parameters +- [patterns.md](./patterns.md) - Common workflows, testing, orchestration +- [gotchas.md](./gotchas.md) - Timeouts, limits, debugging strategies + +## See Also +- [durable-objects](../durable-objects/) - Alternative stateful approach +- [queues](../queues/) - Message-driven workflows +- [workers](../workers/) - Entry point for workflow instances diff --git a/plugins/cloudflare/skills/cloudflare/references/workflows/api.md b/plugins/cloudflare/skills/cloudflare/references/workflows/api.md new file mode 100644 index 0000000..7ac5625 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workflows/api.md @@ -0,0 +1,185 @@ +# Workflow APIs + +## Step APIs + +```typescript +// step.do() +const result = await step.do('step name', async () => { /* logic */ }); +const result = await step.do('step name', { retries, timeout }, async () => {}); + +// step.sleep() +await step.sleep('description', '1 hour'); +await step.sleep('description', 5000); // ms + +// step.sleepUntil() +await step.sleepUntil('description', Date.parse('2024-12-31')); + +// step.waitForEvent() +const data = await step.waitForEvent('wait', {event: 'webhook-type', timeout: '24h'}); // Default 24h, max 365d +try { const event = await step.waitForEvent('wait', { event: 'approval', timeout: '1h' }); } catch (e) { /* Timeout */ } +``` + +## Instance Management + +```typescript +// Create single +const instance = await env.MY_WORKFLOW.create({id: crypto.randomUUID(), params: { userId: 'user123' }}); // id optional, auto-generated if omitted + +// Create with custom retention (default: 3 days free, 30 days paid) +const instance = await env.MY_WORKFLOW.create({ + id: crypto.randomUUID(), + params: { userId: 'user123' }, + retention: '30 days' // Override default retention period +}); + +// Batch (max 100, idempotent: skips existing IDs) +const instances = await env.MY_WORKFLOW.createBatch([{id: 'user1', params: {name: 'John'}}, {id: 'user2', params: {name: 'Jane'}}]); + +// Get & Status +const instance = await env.MY_WORKFLOW.get('instance-id'); +const status = await instance.status(); // {status: 'queued' | 'running' | 'paused' | 'errored' | 'terminated' | 'complete' | 'waiting' | 'waitingForPause' | 'unknown', error?, output?} + +// Control +await instance.pause(); await instance.resume(); await instance.terminate(); await instance.restart(); + +// Send Events +await instance.sendEvent({type: 'approval', payload: { approved: true }}); // Must match waitForEvent type +``` + +## Triggering Workflows + +```typescript +// From Worker +export default { async fetch(req, env) { const instance = await env.MY_WORKFLOW.create({id: crypto.randomUUID(), params: { userId: 'user123' }}); return Response.json({ id: instance.id }); }}; + +// From Queue +export default { async queue(batch, env) { for (const msg of batch.messages) { await env.MY_WORKFLOW.create({id: `job-${msg.id}`, params: msg.body}); } }}; + +// From Cron +export default { async scheduled(event, env) { await env.CLEANUP_WORKFLOW.create({id: `cleanup-${Date.now()}`, params: { timestamp: event.scheduledTime }}); }}; + +// From Another Workflow (non-blocking) +export class ParentWorkflow extends WorkflowEntrypoint { + async run(event, step) { + const child = await step.do('start child', async () => await this.env.CHILD_WORKFLOW.create({id: `child-${event.instanceId}`, params: {}})); + } +} +``` + +## Error Handling + +```typescript +import { NonRetryableError } from 'cloudflare:workers'; + +// NonRetryableError +await step.do('validate', async () => { + if (!event.params.paymentMethod) throw new NonRetryableError('Payment method required'); + const res = await fetch('https://api.example.com/charge', { method: 'POST' }); + if (res.status === 401) throw new NonRetryableError('Invalid credentials'); // Don't retry + if (!res.ok) throw new Error('Retryable failure'); // Will retry + return res.json(); +}); + +// Catching Errors +try { await step.do('risky op', async () => { throw new NonRetryableError('Failed'); }); } catch (e) { await step.do('cleanup', async () => {}); } + +// Idempotency +await step.do('charge', async () => { + const sub = await fetch(`https://api/subscriptions/${id}`).then(r => r.json()); + if (sub.charged) return sub; // Already done + return await fetch(`https://api/subscriptions/${id}`, {method: 'POST', body: JSON.stringify({ amount: 10.0 })}).then(r => r.json()); +}); +``` + +## Type Constraints + +Params and step returns must be `Rpc.Serializable`: + +```typescript +// ✅ Valid types +type ValidParams = { + userId: string; + count: number; + tags: string[]; + metadata: Record; +}; + +// ❌ Invalid types +type InvalidParams = { + callback: () => void; // Functions not serializable + symbol: symbol; // Symbols not serializable + circular: any; // Circular references not allowed +}; + +// Step returns follow same rules +const result = await step.do('fetch', async () => { + return { userId: '123', data: [1, 2, 3] }; // ✅ Plain object +}); +``` + +## Sleep & Scheduling + +```typescript +// Relative +await step.sleep('wait 1 hour', '1 hour'); +await step.sleep('wait 30 days', '30 days'); +await step.sleep('wait 5s', 5000); // ms + +// Absolute +await step.sleepUntil('launch date', Date.parse('24 Oct 2024 13:00:00 UTC')); +await step.sleepUntil('deadline', new Date('2024-12-31T23:59:59Z')); +``` + +Units: second, minute, hour, day, week, month, year. Max: 365 days. +Sleeping instances don't count toward concurrency. + +## Parameters + +**Pass from Worker:** +```typescript +const instance = await env.MY_WORKFLOW.create({ + id: crypto.randomUUID(), + params: { userId: 'user123', email: 'user@example.com' } +}); +``` + +**Access in Workflow:** +```typescript +async run(event: WorkflowEvent, step: WorkflowStep) { + const userId = event.params.userId; + const instanceId = event.instanceId; + const createdAt = event.timestamp; +} +``` + +**CLI Trigger:** +```bash +npx wrangler workflows trigger my-workflow '{"userId":"user123"}' +``` + +## Wrangler CLI + +```bash +npm create cloudflare@latest my-workflow -- --template "cloudflare/workflows-starter" +npx wrangler deploy +npx wrangler workflows list +npx wrangler workflows trigger my-workflow '{"userId":"user123"}' +npx wrangler workflows instances list my-workflow +npx wrangler workflows instances describe my-workflow instance-id +npx wrangler workflows instances pause/resume/terminate my-workflow instance-id +``` + +## REST API + +```bash +# Create +curl -X POST "https://api.cloudflare.com/client/v4/accounts/{account_id}/workflows/{workflow_name}/instances" -H "Authorization: Bearer {token}" -d '{"id":"custom-id","params":{"userId":"user123"}}' + +# Status +curl "https://api.cloudflare.com/client/v4/accounts/{account_id}/workflows/{workflow_name}/instances/{instance_id}/status" -H "Authorization: Bearer {token}" + +# Send Event +curl -X POST "https://api.cloudflare.com/client/v4/accounts/{account_id}/workflows/{workflow_name}/instances/{instance_id}/events" -H "Authorization: Bearer {token}" -d '{"type":"approval","payload":{"approved":true}}' +``` + +See: [configuration.md](./configuration.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workflows/configuration.md b/plugins/cloudflare/skills/cloudflare/references/workflows/configuration.md new file mode 100644 index 0000000..cfe7e4a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workflows/configuration.md @@ -0,0 +1,151 @@ +# Workflow Configuration + +## wrangler.jsonc Setup + +```jsonc +{ + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date for new projects + "observability": { + "enabled": true // Enables Workflows dashboard + structured logs + }, + "workflows": [ + { + "name": "my-workflow", // Workflow name + "binding": "MY_WORKFLOW", // Env binding + "class_name": "MyWorkflow" // TS class name + // "script_name": "other-worker" // For cross-script calls + } + ], + "limits": { + "cpu_ms": 300000 // 5 min max (default 30s) + } +} +``` + +## Step Configuration + +```typescript +// Basic step +const data = await step.do('step name', async () => ({ result: 'value' })); + +// With retry config +await step.do('api call', { + retries: { + limit: 10, // Default: 5, or Infinity + delay: '10 seconds', // Default: 10000ms + backoff: 'exponential' // constant | linear | exponential + }, + timeout: '30 minutes' // Per-attempt timeout (default: 10min) +}, async () => { + const res = await fetch('https://api.example.com/data'); + if (!res.ok) throw new Error('Failed'); + return res.json(); +}); +``` + +### Parallel Steps +```typescript +const [user, settings] = await Promise.all([ + step.do('fetch user', async () => this.env.KV.get(`user:${id}`)), + step.do('fetch settings', async () => this.env.KV.get(`settings:${id}`)) +]); +``` + +### Conditional Steps +```typescript +const config = await step.do('fetch config', async () => + this.env.KV.get('flags', { type: 'json' }) +); + +// ✅ Deterministic (based on step output) +if (config.enableEmail) { + await step.do('send email', async () => sendEmail()); +} + +// ❌ Non-deterministic (Date.now outside step) +if (Date.now() > deadline) { /* BAD */ } +``` + +### Dynamic Steps (Loops) +```typescript +const files = await step.do('list files', async () => + this.env.BUCKET.list() +); + +for (const file of files.objects) { + await step.do(`process ${file.key}`, async () => { + const obj = await this.env.BUCKET.get(file.key); + return processData(await obj.arrayBuffer()); + }); +} +``` + +## Multiple Workflows + +```jsonc +{ + "workflows": [ + {"name": "user-onboarding", "binding": "USER_ONBOARDING", "class_name": "UserOnboarding"}, + {"name": "data-processing", "binding": "DATA_PROCESSING", "class_name": "DataProcessing"} + ] +} +``` + +Each class extends `WorkflowEntrypoint` with its own `Params` type. + +## Cross-Script Bindings + +Worker A defines workflow. Worker B calls it by adding `script_name`: + +```jsonc +// Worker B (caller) +{ + "workflows": [{ + "name": "billing-workflow", + "binding": "BILLING", + "script_name": "billing-worker" // Points to Worker A + }] +} +``` + +## Bindings + +Workflows access Cloudflare bindings via `this.env`: + +```typescript +type Env = { + MY_WORKFLOW: Workflow; + KV: KVNamespace; + DB: D1Database; + BUCKET: R2Bucket; + AI: Ai; + VECTORIZE: VectorizeIndex; +}; + +await step.do('use bindings', async () => { + const kv = await this.env.KV.get('key'); + const db = await this.env.DB.prepare('SELECT * FROM users').first(); + const file = await this.env.BUCKET.get('file.txt'); + const ai = await this.env.AI.run('@cf/meta/llama-2-7b-chat-int8', { prompt: 'Hi' }); +}); +``` + +## Pages Functions Binding + +Pages Functions can trigger Workflows via service bindings: + +```typescript +// functions/_middleware.ts +export const onRequest: PagesFunction = async ({ env, request }) => { + const instance = await env.MY_WORKFLOW.create({ + params: { url: request.url } + }); + return new Response(`Started ${instance.id}`); +}; +``` + +Configure in wrangler.jsonc under `service_bindings`. + +See: [api.md](./api.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workflows/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/workflows/gotchas.md new file mode 100644 index 0000000..6f85444 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workflows/gotchas.md @@ -0,0 +1,97 @@ +# Gotchas & Debugging + +## Common Errors + +### "Step Timeout" + +**Cause:** Step execution exceeding 10 minute default timeout or configured timeout +**Solution:** Set custom timeout with `step.do('long operation', {timeout: '30 minutes'}, async () => {...})` or increase CPU limit in wrangler.jsonc (max 5min CPU time) + +### "waitForEvent Timeout" + +**Cause:** Event not received within timeout period (default 24h, max 365d) +**Solution:** Wrap in try-catch to handle timeout gracefully and proceed with default behavior + +### "Non-Deterministic Step Names" + +**Cause:** Using dynamic values like `Date.now()` in step names causes replay issues +**Solution:** Use deterministic values like `event.instanceId` for step names + +### "State Lost in Variables" + +**Cause:** Using module-level or local variables to store state which is lost on hibernation +**Solution:** Return values from `step.do()` which are automatically persisted: `const total = await step.do('step 1', async () => 10)` + +### "Non-Deterministic Conditionals" + +**Cause:** Using non-deterministic logic (like `Date.now()`) outside steps in conditionals +**Solution:** Move non-deterministic operations inside steps: `const isLate = await step.do('check', async () => Date.now() > deadline)` + +### "Large Step Returns Exceeding Limit" + +**Cause:** Returning data >1 MiB from step +**Solution:** Store large data in R2 and return only reference: `{ key: 'r2-object-key' }` + +### "Step Exceeded CPU Limit But Ran for < 30s" + +**Cause:** Confusion between CPU time (active compute) and wall-clock time (includes I/O waits) +**Solution:** Network requests, database queries, and sleeps don't count toward CPU. 30s limit = 30s of active processing + +### "Idempotency Violation" + +**Cause:** Step operations not idempotent, causing duplicate charges or actions on retry +**Solution:** Check if operation already completed before executing (e.g., check if customer already charged) + +### "Instance ID Collision" + +**Cause:** Reusing instance IDs causing conflicts +**Solution:** Use unique IDs with timestamp: `await env.MY_WORKFLOW.create({ id: \`${userId}-${Date.now()}\`, params: {} })` + +### "Instance Data Disappeared After Completion" + +**Cause:** Completed/errored instances are automatically deleted after retention period (3 days free / 30 days paid) +**Solution:** Export critical data to KV/R2/D1 before workflow completes + +### "Missing await on step.do" + +**Cause:** Forgetting to await step.do() causing fire-and-forget behavior +**Solution:** Always await step operations: `await step.do('task', ...)` + +## Limits + +| Limit | Free | Paid | Notes | +|-------|------|------|-------| +| CPU per step | 10ms | 30s (default), 5min (max) | Set via `limits.cpu_ms` in wrangler.jsonc | +| Step state | 1 MiB | 1 MiB | Per step return value | +| Instance state | 100 MB | 1 GB | Total state per workflow instance | +| Steps per workflow | 1,024 | 1,024 | `step.sleep()` doesn't count | +| Executions per day | 100k | Unlimited | Daily execution limit | +| Concurrent instances | 25 | 10k | Maximum concurrent workflows; waiting state excluded | +| Queued instances | 100k | 1M | Maximum queued workflow instances | +| Subrequests per step | 50 | 1,000 | Maximum outbound requests per step | +| State retention | 3 days | 30 days | How long completed instances kept | +| Step timeout default | 10 min | 10 min | Per attempt | +| waitForEvent timeout default | 24h | 24h | Maximum 365 days | +| waitForEvent timeout max | 365 days | 365 days | Maximum wait time | + +**Note:** Instances in `waiting` state (from `step.sleep` or `step.waitForEvent`) don't count toward concurrent instance limit, allowing millions of sleeping workflows. + +## Pricing + +| Metric | Free | Paid | Notes | +|--------|------|------|-------| +| Requests | 100k/day | 10M/mo + $0.30/M | Workflow invocations | +| CPU time | 10ms/invoke | 30M CPU-ms/mo + $0.02/M CPU-ms | Actual CPU usage | +| Storage | 1 GB | 1 GB/mo + $0.20/GB-mo | All instances (running/errored/sleeping/completed) | + +## References + +- [Official Docs](https://developers.cloudflare.com/workflows/) +- [Get Started Guide](https://developers.cloudflare.com/workflows/get-started/guide/) +- [Workers API](https://developers.cloudflare.com/workflows/build/workers-api/) +- [REST API](https://developers.cloudflare.com/api/resources/workflows/) +- [Examples](https://developers.cloudflare.com/workflows/examples/) +- [Limits](https://developers.cloudflare.com/workflows/reference/limits/) +- [Pricing](https://developers.cloudflare.com/workflows/reference/pricing/) + +See: [README.md](./README.md), [configuration.md](./configuration.md), [api.md](./api.md), [patterns.md](./patterns.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/workflows/patterns.md b/plugins/cloudflare/skills/cloudflare/references/workflows/patterns.md new file mode 100644 index 0000000..72ce024 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/workflows/patterns.md @@ -0,0 +1,175 @@ +# Workflow Patterns + +## Image Processing Pipeline + +```typescript +export class ImageProcessingWorkflow extends WorkflowEntrypoint { + async run(event, step) { + const imageData = await step.do('fetch', async () => (await this.env.BUCKET.get(event.params.imageKey)).arrayBuffer()); + const description = await step.do('generate description', async () => + await this.env.AI.run('@cf/llava-hf/llava-1.5-7b-hf', {image: Array.from(new Uint8Array(imageData)), prompt: 'Describe this image', max_tokens: 50}) + ); + await step.waitForEvent('await approval', { event: 'approved', timeout: '24h' }); + await step.do('publish', async () => await this.env.BUCKET.put(`public/${event.params.imageKey}`, imageData)); + } +} +``` + +## User Lifecycle + +```typescript +export class UserLifecycleWorkflow extends WorkflowEntrypoint { + async run(event, step) { + await step.do('welcome email', async () => await sendEmail(event.params.email, 'Welcome!')); + await step.sleep('trial period', '7 days'); + const hasConverted = await step.do('check conversion', async () => { + const user = await this.env.DB.prepare('SELECT subscription_status FROM users WHERE id = ?').bind(event.params.userId).first(); + return user.subscription_status === 'active'; + }); + if (!hasConverted) await step.do('trial expiration email', async () => await sendEmail(event.params.email, 'Trial ending')); + } +} +``` + +## Data Pipeline + +```typescript +export class DataPipelineWorkflow extends WorkflowEntrypoint { + async run(event, step) { + const rawData = await step.do('extract', {retries: { limit: 10, delay: '30s', backoff: 'exponential' }}, async () => { + const res = await fetch(event.params.sourceUrl); + if (!res.ok) throw new Error('Fetch failed'); + return res.json(); + }); + const transformed = await step.do('transform', async () => + rawData.map(item => ({ id: item.id, normalized: normalizeData(item) })) + ); + const dataRef = await step.do('store', async () => { + const key = `processed/${Date.now()}.json`; + await this.env.BUCKET.put(key, JSON.stringify(transformed)); + return { key }; + }); + await step.do('load', async () => { + const data = await (await this.env.BUCKET.get(dataRef.key)).json(); + for (let i = 0; i < data.length; i += 100) { + await this.env.DB.batch(data.slice(i, i + 100).map(item => + this.env.DB.prepare('INSERT INTO records VALUES (?, ?)').bind(item.id, item.normalized) + )); + } + }); + } +} +``` + +## Human-in-the-Loop Approval + +```typescript +export class ApprovalWorkflow extends WorkflowEntrypoint { + async run(event, step) { + await step.do('create approval', async () => await this.env.DB.prepare('INSERT INTO approvals (id, user_id, status) VALUES (?, ?, ?)').bind(event.instanceId, event.params.userId, 'pending').run()); + try { + const approval = await step.waitForEvent<{ approved: boolean }>('wait for approval', { event: 'approval-response', timeout: '48h' }); + if (approval.approved) { await step.do('process approval', async () => {}); } + else { await step.do('handle rejection', async () => {}); } + } catch (e) { + await step.do('auto reject', async () => await this.env.DB.prepare('UPDATE approvals SET status = ? WHERE id = ?').bind('auto-rejected', event.instanceId).run()); + } + } +} +``` + +## Testing Workflows + +### Setup + +```typescript +// vitest.config.ts +import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config'; + +export default defineWorkersConfig({ + test: { + poolOptions: { + workers: { + wrangler: { configPath: './wrangler.jsonc' } + } + } + } +}); +``` + +### Introspection API + +```typescript +import { introspectWorkflowInstance } from 'cloudflare:test'; + +const instance = await env.MY_WORKFLOW.create({ params: { userId: '123' } }); +const introspector = await introspectWorkflowInstance(env.MY_WORKFLOW, instance.id); + +// Wait for step completion +const result = await introspector.waitForStepResult({ name: 'fetch user', index: 0 }); + +// Mock step behavior +await introspector.modify(async (m) => { + await m.mockStepResult({ name: 'api call' }, { mocked: true }); +}); +``` + +## Best Practices + +### ✅ DO + +1. **Granular steps**: One API call per step (unless proving idempotency) +2. **Idempotency**: Check-then-execute; use idempotency keys +3. **Deterministic names**: Use static or step-output-based names +4. **Return state**: Persist via step returns, not variables +5. **Always await**: `await step.do()`, avoid dangling promises +6. **Deterministic conditionals**: Base on `event.payload` or step outputs +7. **Store large data externally**: R2/KV for >1 MiB, return refs +8. **Batch creation**: `createBatch()` for multiple instances + +### ❌ DON'T + +1. **One giant step**: Breaks durability & retry control +2. **State outside steps**: Lost on hibernation +3. **Mutate events**: Events immutable, return new state +4. **Non-deterministic logic outside steps**: `Math.random()`, `Date.now()` must be in steps +5. **Side effects outside steps**: May duplicate on restart +6. **Non-deterministic step names**: Prevents caching +7. **Ignore timeouts**: `waitForEvent` throws, use try-catch +8. **Reuse instance IDs**: Must be unique within retention + +## Orchestration Patterns + +### Fan-Out (Parallel Processing) +```typescript +const files = await step.do('list', async () => this.env.BUCKET.list()); +await Promise.all(files.objects.map((file, i) => step.do(`process ${i}`, async () => processFile(await (await this.env.BUCKET.get(file.key)).arrayBuffer())))); +``` + +### Parent-Child Workflows +```typescript +const child = await step.do('start child', async () => await this.env.CHILD_WORKFLOW.create({id: `child-${event.instanceId}`, params: { data: result.data }})); +await step.do('other work', async () => console.log(`Child started: ${child.id}`)); +``` + +### Race Pattern +```typescript +const winner = await Promise.race([ + step.do('option A', async () => slowOperation()), + step.do('option B', async () => fastOperation()) +]); +``` + +### Scheduled Workflow Chain +```typescript +export default { async scheduled(event, env) { await env.DAILY_WORKFLOW.create({id: `daily-${event.scheduledTime}`, params: { timestamp: event.scheduledTime }}); }}; +export class DailyWorkflow extends WorkflowEntrypoint { + async run(event, step) { + await step.do('daily task', async () => {}); + await step.sleep('wait 7 days', '7 days'); + await step.do('weekly followup', async () => {}); + } +} +``` + +See: [configuration.md](./configuration.md), [api.md](./api.md), [gotchas.md](./gotchas.md) diff --git a/plugins/cloudflare/skills/cloudflare/references/wrangler/README.md b/plugins/cloudflare/skills/cloudflare/references/wrangler/README.md new file mode 100644 index 0000000..0640e7c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/wrangler/README.md @@ -0,0 +1,141 @@ +# Cloudflare Wrangler + +Official CLI for Cloudflare Workers - develop, manage, and deploy Workers from the command line. + +## What is Wrangler? + +Wrangler is the Cloudflare Developer Platform CLI that allows you to: +- Create, develop, and deploy Workers +- Manage bindings (KV, D1, R2, Durable Objects, etc.) +- Configure routing and environments +- Run local development servers +- Execute migrations and manage resources +- Perform integration testing + +## Installation + +```bash +npm install wrangler --save-dev +# or globally +npm install -g wrangler +``` + +Run commands: `npx wrangler ` (or `pnpm`/`yarn wrangler`) + +## Reading Order + +| If you want to... | Start here | +|-------------------|------------| +| Create/deploy Worker quickly | Essential Commands below → [patterns.md](./patterns.md) §New Worker | +| Configure bindings (KV, D1, R2) | [configuration.md](./configuration.md) §Bindings | +| Write integration tests | [api.md](./api.md) §startWorker | +| Debug production issues | [gotchas.md](./gotchas.md) + Essential Commands §Monitoring | +| Set up multi-environment workflow | [configuration.md](./configuration.md) §Environments | + +## Essential Commands + +### Project & Development +```bash +wrangler init [name] # Create new project +wrangler dev # Local dev server (fast, simulated) +wrangler dev --remote # Dev with remote resources (production-like) +wrangler deploy # Deploy to production +wrangler deploy --env staging # Deploy to environment +wrangler versions list # List versions +wrangler rollback [id] # Rollback deployment +wrangler login # OAuth login +wrangler whoami # Check auth status +``` + +## Resource Management + +### KV +```bash +wrangler kv namespace create NAME +wrangler kv key put "key" "value" --namespace-id= +wrangler kv key get "key" --namespace-id= +``` + +### D1 +```bash +wrangler d1 create NAME +wrangler d1 execute NAME --command "SQL" +wrangler d1 migrations create NAME "description" +wrangler d1 migrations apply NAME +``` + +### R2 +```bash +wrangler r2 bucket create NAME +wrangler r2 object put BUCKET/key --file path +wrangler r2 object get BUCKET/key +``` + +### Other Resources +```bash +wrangler queues create NAME +wrangler vectorize create NAME --dimensions N --metric cosine +wrangler hyperdrive create NAME --connection-string "..." +wrangler workflows create NAME +wrangler constellation create NAME +wrangler pages project create NAME +wrangler pages deployment create --project NAME --branch main +``` + +### Secrets +```bash +wrangler secret put NAME # Set Worker secret +wrangler secret list # List Worker secrets +wrangler secret delete NAME # Delete Worker secret +wrangler secret bulk FILE.json # Bulk upload from JSON + +# Secrets Store (centralized, reusable across Workers) +wrangler secret-store:secret put STORE_NAME SECRET_NAME +wrangler secret-store:secret list STORE_NAME +``` + +### Monitoring +```bash +wrangler tail # Real-time logs +wrangler tail --env production # Tail specific env +wrangler tail --status error # Filter by status +``` + +## In This Reference + +- [auth.md](./auth.md) - Authentication setup (`wrangler login`, API tokens) +- [configuration.md](./configuration.md) - wrangler.jsonc setup, environments, bindings +- [api.md](./api.md) - Programmatic API (`startWorker`, `getPlatformProxy`, events) +- [patterns.md](./patterns.md) - Common workflows and development patterns +- [gotchas.md](./gotchas.md) - Common pitfalls, limits, and troubleshooting + +## Quick Decision Tree + +``` +Need to test your Worker? +├─ Testing full Worker with bindings → api.md §startWorker +├─ Testing individual functions → api.md §getPlatformProxy +└─ Testing with Vitest → patterns.md §Testing with Vitest + +Need to configure something? +├─ Bindings (KV, D1, R2, etc.) → configuration.md §Bindings +├─ Multiple environments → configuration.md §Environments +├─ Static files → configuration.md §Workers Assets +└─ Routing → configuration.md §Routing + +Development not working? +├─ Local differs from production → Use `wrangler dev --remote` +├─ Bindings not available → gotchas.md §Binding Not Available +└─ Auth issues → auth.md + +Authentication issues? +├─ "Not logged in" / "Unauthorized" → auth.md +├─ First time deploying → `wrangler login` (one-time OAuth) +└─ CI/CD setup → auth.md §API Token +``` + +## See Also + +- [workers](../workers/) - Workers runtime API reference +- [miniflare](../miniflare/) - Local testing with Miniflare +- [workerd](../workerd/) - Runtime that powers `wrangler dev` diff --git a/plugins/cloudflare/skills/cloudflare/references/wrangler/api.md b/plugins/cloudflare/skills/cloudflare/references/wrangler/api.md new file mode 100644 index 0000000..11384c2 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/wrangler/api.md @@ -0,0 +1,188 @@ +# Wrangler Programmatic API + +Node.js APIs for testing and development. + +## startWorker (Testing) + +Starts Worker with real local bindings for integration tests. Stable API (replaces `unstable_startWorker`). + +```typescript +import { startWorker } from "wrangler"; +import { describe, it, before, after } from "node:test"; +import assert from "node:assert"; + +describe("worker", () => { + let worker; + + before(async () => { + worker = await startWorker({ + config: "wrangler.jsonc", + environment: "development" + }); + }); + + after(async () => { + await worker.dispose(); + }); + + it("responds with 200", async () => { + const response = await worker.fetch("http://example.com"); + assert.strictEqual(response.status, 200); + }); +}); +``` + +### Options + +| Option | Type | Description | +|--------|------|-------------| +| `config` | `string` | Path to wrangler.jsonc | +| `environment` | `string` | Environment name from config | +| `persist` | `boolean \| { path: string }` | Enable persistent state | +| `bundle` | `boolean` | Enable bundling (default: true) | +| `remote` | `false \| true \| "minimal"` | Remote mode: `false` (local), `true` (full remote), `"minimal"` (remote bindings only) | + +### Remote Mode + +```typescript +// Local mode (default) - fast, simulated +const worker = await startWorker({ config: "wrangler.jsonc" }); + +// Full remote mode - production-like, slower +const worker = await startWorker({ + config: "wrangler.jsonc", + remote: true +}); + +// Minimal remote mode - remote bindings, local Worker +const worker = await startWorker({ + config: "wrangler.jsonc", + remote: "minimal" +}); +``` + +## getPlatformProxy + +Emulate bindings in Node.js without starting Worker. + +```typescript +import { getPlatformProxy } from "wrangler"; + +const { env, dispose, caches } = await getPlatformProxy({ + configPath: "wrangler.jsonc", + environment: "production", + persist: { path: ".wrangler/state" } +}); + +// Use bindings +const value = await env.MY_KV.get("key"); +await env.DB.prepare("SELECT * FROM users").all(); +await env.ASSETS.put("file.txt", "content"); + +// Platform APIs +await caches.default.put("https://example.com", new Response("cached")); + +await dispose(); +``` + +Use for unit tests (test functions, not full Worker) or scripts that need bindings. + +## Type Generation + +Generate types from config: `wrangler types` → creates `worker-configuration.d.ts` + +## Event System + +Listen to Worker lifecycle events for advanced workflows. + +```typescript +import { startWorker } from "wrangler"; + +const worker = await startWorker({ + config: "wrangler.jsonc", + bundle: true +}); + +// Bundle events +worker.on("bundleStart", (details) => { + console.log("Bundling started:", details.config); +}); + +worker.on("bundleComplete", (details) => { + console.log("Bundle ready:", details.duration); +}); + +// Reconfiguration events +worker.on("reloadStart", () => { + console.log("Worker reloading..."); +}); + +worker.on("reloadComplete", () => { + console.log("Worker reloaded"); +}); + +await worker.dispose(); +``` + +### Dynamic Reconfiguration + +```typescript +import { startWorker } from "wrangler"; + +const worker = await startWorker({ config: "wrangler.jsonc" }); + +// Replace entire config +await worker.setConfig({ + config: "wrangler.staging.jsonc", + environment: "staging" +}); + +// Patch specific fields +await worker.patchConfig({ + vars: { DEBUG: "true" } +}); + +await worker.dispose(); +``` + +## unstable_dev (Deprecated) + +Use `startWorker` instead. + +## Multi-Worker Registry + +Test multiple Workers with service bindings. + +```typescript +import { startWorker } from "wrangler"; + +const auth = await startWorker({ config: "./auth/wrangler.jsonc" }); +const api = await startWorker({ + config: "./api/wrangler.jsonc", + bindings: { AUTH: auth } // Service binding +}); + +const response = await api.fetch("http://example.com/api/login"); +// API Worker calls AUTH Worker via env.AUTH.fetch() + +await api.dispose(); +await auth.dispose(); +``` + +## Best Practices + +- Use `startWorker` for integration tests (tests full Worker) +- Use `getPlatformProxy` for unit tests (tests individual functions) +- Use `remote: true` when debugging production-specific issues +- Use `remote: "minimal"` for faster tests with real bindings +- Enable `persist: true` for debugging (state survives runs) +- Run `wrangler types` after config changes +- Always `dispose()` to prevent resource leaks +- Listen to bundle events for build monitoring +- Use multi-worker registry for testing service bindings + +## See Also + +- [README.md](./README.md) - CLI commands +- [configuration.md](./configuration.md) - Config +- [patterns.md](./patterns.md) - Testing patterns diff --git a/plugins/cloudflare/skills/cloudflare/references/wrangler/configuration.md b/plugins/cloudflare/skills/cloudflare/references/wrangler/configuration.md new file mode 100644 index 0000000..20dc2f0 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/wrangler/configuration.md @@ -0,0 +1,197 @@ +# Wrangler Configuration + +Configuration reference for wrangler.jsonc (recommended). + +## Config Format + +**wrangler.jsonc recommended** (v3.91.0+) - provides schema validation. + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2025-01-01", // Use current date + "vars": { "API_KEY": "dev-key" }, + "kv_namespaces": [{ "binding": "MY_KV", "id": "abc123" }] +} +``` + +## Field Inheritance + +Inheritable: `name`, `main`, `compatibility_date`, `routes`, `triggers` +Non-inheritable (define per env): `vars`, bindings (KV, D1, R2, etc.) + +## Environments + +```jsonc +{ + "name": "my-worker", + "vars": { "ENV": "dev" }, + "env": { + "production": { + "name": "my-worker-prod", + "vars": { "ENV": "prod" }, + "route": { "pattern": "example.com/*", "zone_name": "example.com" } + } + } +} +``` + +Deploy: `wrangler deploy --env production` + +## Routing + +```jsonc +// Custom domain (recommended) +{ "routes": [{ "pattern": "api.example.com", "custom_domain": true }] } + +// Zone-based +{ "routes": [{ "pattern": "api.example.com/*", "zone_name": "example.com" }] } + +// workers.dev +{ "workers_dev": true } +``` + +## Bindings + +```jsonc +// Variables +{ "vars": { "API_URL": "https://api.example.com" } } + +// KV +{ "kv_namespaces": [{ "binding": "CACHE", "id": "abc123" }] } + +// D1 +{ "d1_databases": [{ "binding": "DB", "database_id": "abc-123" }] } + +// R2 +{ "r2_buckets": [{ "binding": "ASSETS", "bucket_name": "my-assets" }] } + +// Durable Objects +{ "durable_objects": { + "bindings": [{ + "name": "COUNTER", + "class_name": "Counter", + "script_name": "my-worker" // Required for external DOs + }] +} } +{ "migrations": [{ "tag": "v1", "new_sqlite_classes": ["Counter"] }] } + +// Service Bindings +{ "services": [{ "binding": "AUTH", "service": "auth-worker" }] } + +// Queues +{ "queues": { + "producers": [{ "binding": "TASKS", "queue": "task-queue" }], + "consumers": [{ "queue": "task-queue", "max_batch_size": 10 }] +} } + +// Vectorize +{ "vectorize": [{ "binding": "VECTORS", "index_name": "embeddings" }] } + +// Hyperdrive (requires nodejs_compat_v2 for pg/postgres) +{ "hyperdrive": [{ "binding": "HYPERDRIVE", "id": "hyper-id" }] } +{ "compatibility_flags": ["nodejs_compat_v2"] } // For pg/postgres + +// Workers AI +{ "ai": { "binding": "AI" } } + +// Workflows +{ "workflows": [{ "binding": "WORKFLOW", "name": "my-workflow", "class_name": "MyWorkflow" }] } + +// Secrets Store (centralized secrets) +{ "secrets_store": [{ "binding": "SECRETS", "id": "store-id" }] } + +// Constellation (AI inference) +{ "constellation": [{ "binding": "MODEL", "project_id": "proj-id" }] } +``` + +## Workers Assets (Static Files) + +Recommended for serving static files (replaces old `site` config). + +```jsonc +{ + "assets": { + "directory": "./public", + "binding": "ASSETS", + "html_handling": "auto-trailing-slash", // or "none", "force-trailing-slash" + "not_found_handling": "single-page-application" // or "404-page", "none" + } +} +``` + +Access in Worker: +```typescript +export default { + async fetch(request, env) { + // Try serving static asset first + const asset = await env.ASSETS.fetch(request); + if (asset.status !== 404) return asset; + + // Custom logic for non-assets + return new Response("API response"); + } +} +``` + +## Placement + +Control where Workers run geographically. + +```jsonc +{ + "placement": { + "mode": "smart" // or "off" + } +} +``` + +- `"smart"`: Run Worker near data sources (D1, Durable Objects) to reduce latency +- `"off"`: Default distribution (run everywhere) + +## Auto-Provisioning (Beta) + +Omit resource IDs - Wrangler creates them and writes back to config on deploy. + +```jsonc +{ "kv_namespaces": [{ "binding": "MY_KV" }] } // No id - auto-provisioned +``` + +After deploy, ID is added to config automatically. + +## Advanced + +```jsonc +// Cron Triggers +{ "triggers": { "crons": ["0 0 * * *"] } } + +// Observability (tracing) +{ "observability": { "enabled": true, "head_sampling_rate": 0.1 } } + +// Runtime Limits +{ "limits": { "cpu_ms": 100 } } + +// Browser Rendering +{ "browser": { "binding": "BROWSER" } } + +// mTLS Certificates +{ "mtls_certificates": [{ "binding": "CERT", "certificate_id": "cert-uuid" }] } + +// Logpush (stream logs to R2/S3) +{ "logpush": true } + +// Tail Consumers (process logs with another Worker) +{ "tail_consumers": [{ "service": "log-worker" }] } + +// Unsafe bindings (access to arbitrary bindings) +{ "unsafe": { "bindings": [{ "name": "MY_BINDING", "type": "plain_text", "text": "value" }] } } +``` + +## See Also + +- [README.md](./README.md) - Overview and commands +- [api.md](./api.md) - Programmatic API +- [patterns.md](./patterns.md) - Workflows +- [gotchas.md](./gotchas.md) - Common issues diff --git a/plugins/cloudflare/skills/cloudflare/references/wrangler/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/wrangler/gotchas.md new file mode 100644 index 0000000..db6ba08 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/wrangler/gotchas.md @@ -0,0 +1,197 @@ +# Wrangler Common Issues + +## Common Errors + +### "Binding ID vs name mismatch" + +**Cause:** Confusion between binding name (code) and resource ID +**Solution:** Bindings use `binding` (code name) and `id`/`database_id`/`bucket_name` (resource ID). Preview bindings need separate IDs: `preview_id`, `preview_database_id` + +### "Environment not inheriting config" + +**Cause:** Non-inheritable keys not redefined per environment +**Solution:** Non-inheritable keys (bindings, vars) must be redefined per environment. Inheritable keys (routes, compatibility_date) can be overridden + +### "Local dev behavior differs from production" + +**Cause:** Using local simulation instead of remote execution +**Solution:** Choose appropriate remote mode: +- `wrangler dev` (default): Local simulation, fast, limited accuracy +- `wrangler dev --remote`: Full remote execution, production-accurate, slower +- Use `remote: "minimal"` in tests for fast tests with real remote bindings + +### "startWorker doesn't match production" + +**Cause:** Using local mode when remote resources needed +**Solution:** Use `remote` option: +```typescript +const worker = await startWorker({ + config: "wrangler.jsonc", + remote: true // or "minimal" for faster tests +}); +``` + +### "Unexpected runtime changes" + +**Cause:** Missing compatibility_date +**Solution:** Always set `compatibility_date`: +```jsonc +{ "compatibility_date": "2025-01-01" } +``` + +### "Durable Object binding not working" + +**Cause:** Missing script_name for external DOs +**Solution:** Always specify `script_name` for external Durable Objects: +```jsonc +{ + "durable_objects": { + "bindings": [ + { "name": "MY_DO", "class_name": "MyDO", "script_name": "my-worker" } + ] + } +} +``` + +For local DOs in same Worker, `script_name` is optional. + +### "Auto-provisioned resources not appearing" + +**Cause:** IDs written back to config on first deploy, but config not reloaded +**Solution:** After first deploy with auto-provisioning, config file is updated with IDs. Commit the updated config. On subsequent deploys, existing resources are reused. + +### "Secrets not available in local dev" + +**Cause:** Secrets set with `wrangler secret put` only work in deployed Workers +**Solution:** For local dev, use `.dev.vars` + +### "Node.js compatibility error" + +**Cause:** Missing Node.js compatibility flag +**Solution:** Some bindings (Hyperdrive with `pg`) require: +```jsonc +{ "compatibility_flags": ["nodejs_compat_v2"] } +``` + +### "Workers Assets 404 errors" + +**Cause:** Asset path mismatch or incorrect `html_handling` +**Solution:** +- Check `assets.directory` points to correct build output +- Set `html_handling: "auto-trailing-slash"` for SPAs +- Use `not_found_handling: "single-page-application"` to serve index.html for 404s +```jsonc +{ + "assets": { + "directory": "./dist", + "html_handling": "auto-trailing-slash", + "not_found_handling": "single-page-application" + } +} +``` + +### "Placement not reducing latency" + +**Cause:** Misunderstanding of Smart Placement +**Solution:** Smart Placement only helps when Worker accesses D1 or Durable Objects. It doesn't affect KV, R2, or external API latency. +```jsonc +{ "placement": { "mode": "smart" } } // Only beneficial with D1/DOs +``` + +### "unstable_startWorker not found" + +**Cause:** Using outdated API +**Solution:** Use stable `startWorker` instead: +```typescript +import { startWorker } from "wrangler"; // Not unstable_startWorker +``` + +### "outboundService not mocking fetch" + +**Cause:** Mock function not returning Response +**Solution:** Always return Response, use `fetch(req)` for passthrough: +```typescript +const worker = await startWorker({ + outboundService: (req) => { + if (shouldMock(req)) { + return new Response("mocked"); + } + return fetch(req); // Required for non-mocked requests + } +}); +``` + +## Limits + +| Resource/Limit | Value | Notes | +|----------------|-------|-------| +| Bindings per Worker | 64 | Total across all types | +| Environments | Unlimited | Named envs in config | +| Config file size | ~1MB | Keep reasonable | +| Workers Assets size | 25 MB | Per deployment | +| Workers Assets files | 20,000 | Max number of files | +| Script size (compressed) | 1 MB | Free, 10 MB paid | +| CPU time | 10-50ms | Free, 50-500ms paid | +| Subrequest limit | 50 | Free, 1000 paid | + +## Troubleshooting + +### Authentication Issues +```bash +wrangler logout +wrangler login +wrangler whoami +``` + +### Configuration Errors +```bash +wrangler check # Validate config +``` +Use wrangler.jsonc with `$schema` for validation. + +### Binding Not Available +- Check binding exists in config +- For environments, ensure binding defined for that env +- Local dev: some bindings need `--remote` + +### Deployment Failures +```bash +wrangler tail # Check logs +wrangler deploy --dry-run # Validate +wrangler whoami # Check account limits +``` + +### Local Development Issues +```bash +rm -rf .wrangler/state # Clear local state +wrangler dev --remote # Use remote bindings +wrangler dev --persist-to ./local-state # Custom persist location +wrangler dev --inspector-port 9229 # Enable debugging +``` + +### Testing Issues +```bash +# If tests hang, ensure dispose() is called +worker.dispose() // Always cleanup + +# If bindings don't work in tests +const worker = await startWorker({ + config: "wrangler.jsonc", + remote: "minimal" // Use remote bindings +}); +``` + +## Resources + +- Docs: https://developers.cloudflare.com/workers/wrangler/ +- Config: https://developers.cloudflare.com/workers/wrangler/configuration/ +- Commands: https://developers.cloudflare.com/workers/wrangler/commands/ +- Examples: https://github.com/cloudflare/workers-sdk/tree/main/templates +- Discord: https://discord.gg/cloudflaredev + +## See Also + +- [README.md](./README.md) - Commands +- [configuration.md](./configuration.md) - Config +- [api.md](./api.md) - Programmatic API +- [patterns.md](./patterns.md) - Workflows diff --git a/plugins/cloudflare/skills/cloudflare/references/wrangler/patterns.md b/plugins/cloudflare/skills/cloudflare/references/wrangler/patterns.md new file mode 100644 index 0000000..a22a41a --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/wrangler/patterns.md @@ -0,0 +1,209 @@ +# Wrangler Development Patterns + +Common workflows and best practices. + +## New Worker Project + +```bash +wrangler init my-worker && cd my-worker +wrangler dev # Develop locally +wrangler deploy # Deploy +``` + +## Local Development + +```bash +wrangler dev # Local mode (fast, simulated) +wrangler dev --remote # Remote mode (production-accurate) +wrangler dev --env staging --port 8787 +wrangler dev --inspector-port 9229 # Enable debugging +``` + +Debug: chrome://inspect → Configure → localhost:9229 + +## Secrets + +```bash +# Production +echo "secret-value" | wrangler secret put SECRET_KEY + +# Local: use .dev.vars (gitignored) +# SECRET_KEY=local-dev-key +``` + +## Adding KV + +```bash +wrangler kv namespace create MY_KV +wrangler kv namespace create MY_KV --preview +# Add to wrangler.jsonc: { "binding": "MY_KV", "id": "abc123" } +wrangler deploy +``` + +## Adding D1 + +```bash +wrangler d1 create my-db +wrangler d1 migrations create my-db "initial_schema" +# Edit migration file in migrations/, then: +wrangler d1 migrations apply my-db --local +wrangler deploy +wrangler d1 migrations apply my-db --remote + +# Time Travel (restore to point in time) +wrangler d1 time-travel restore my-db --timestamp 2025-01-01T12:00:00Z +``` + +## Multi-Environment + +```bash +wrangler deploy --env staging +wrangler deploy --env production +``` + +```jsonc +{ "env": { "staging": { "vars": { "ENV": "staging" } } } } +``` + +## Testing + +### Integration Tests with Node.js Test Runner + +```typescript +import { startWorker } from "wrangler"; +import { describe, it, before, after } from "node:test"; +import assert from "node:assert"; + +describe("API", () => { + let worker; + + before(async () => { + worker = await startWorker({ + config: "wrangler.jsonc", + remote: "minimal" // Fast tests with real bindings + }); + }); + + after(async () => await worker.dispose()); + + it("creates user", async () => { + const response = await worker.fetch("http://example.com/api/users", { + method: "POST", + body: JSON.stringify({ name: "Alice" }) + }); + assert.strictEqual(response.status, 201); + }); +}); +``` + +### Testing with Vitest + +Install: `npm install -D vitest @cloudflare/vitest-pool-workers` + +**vitest.config.ts:** +```typescript +import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config"; +export default defineWorkersConfig({ + test: { poolOptions: { workers: { wrangler: { configPath: "./wrangler.jsonc" } } } } +}); +``` + +**tests/api.test.ts:** +```typescript +import { env, SELF } from "cloudflare:test"; +import { describe, it, expect } from "vitest"; + +it("fetches users", async () => { + const response = await SELF.fetch("https://example.com/api/users"); + expect(response.status).toBe(200); +}); + +it("uses bindings", async () => { + await env.MY_KV.put("key", "value"); + expect(await env.MY_KV.get("key")).toBe("value"); +}); +``` + +### Multi-Worker Development (Service Bindings) + +```typescript +const authWorker = await startWorker({ config: "./auth/wrangler.jsonc" }); +const apiWorker = await startWorker({ + config: "./api/wrangler.jsonc", + bindings: { AUTH: authWorker } // Service binding +}); + +// Test API calling AUTH +const response = await apiWorker.fetch("http://example.com/api/protected"); +await authWorker.dispose(); +await apiWorker.dispose(); +``` + +### Mock External APIs + +```typescript +const worker = await startWorker({ + config: "wrangler.jsonc", + outboundService: (req) => { + const url = new URL(req.url); + if (url.hostname === "api.external.com") { + return new Response(JSON.stringify({ mocked: true }), { + headers: { "content-type": "application/json" } + }); + } + return fetch(req); // Pass through other requests + } +}); + +// Test Worker that calls external API +const response = await worker.fetch("http://example.com/proxy"); +// Worker internally fetches api.external.com - gets mocked response +``` + +## Monitoring & Versions + +```bash +wrangler tail # Real-time logs +wrangler tail --status error # Filter errors +wrangler versions list +wrangler rollback [id] +``` + +## TypeScript + +```bash +wrangler types # Generate types from config +``` + +```typescript +export default { + async fetch(request: Request, env: Env): Promise { + return Response.json({ value: await env.MY_KV.get("key") }); + } +} satisfies ExportedHandler; +``` + +## Workers Assets + +```jsonc +{ "assets": { "directory": "./dist", "binding": "ASSETS" } } +``` + +```typescript +export default { + async fetch(request, env) { + // API routes first + if (new URL(request.url).pathname.startsWith("/api/")) { + return Response.json({ data: "from API" }); + } + return env.ASSETS.fetch(request); // Static assets + } +} +``` + +## See Also + +- [README.md](./README.md) - Commands +- [configuration.md](./configuration.md) - Config +- [api.md](./api.md) - Programmatic API +- [gotchas.md](./gotchas.md) - Issues diff --git a/plugins/cloudflare/skills/cloudflare/references/zaraz/IMPLEMENTATION_SUMMARY.md b/plugins/cloudflare/skills/cloudflare/references/zaraz/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 0000000..dd8da19 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/zaraz/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,121 @@ +# Zaraz Reference Implementation Summary + +## Files Created + +| File | Lines | Purpose | +|------|-------|---------| +| README.md | 111 | Navigation, decision tree, quick start | +| api.md | 287 | Web API reference, Zaraz Context | +| configuration.md | 307 | Dashboard setup, triggers, tools, consent | +| patterns.md | 430 | SPA, e-commerce, Worker integration | +| gotchas.md | 317 | Troubleshooting, limits, tool-specific issues | +| **Total** | **1,452** | **vs 366 original** | + +## Key Improvements Applied + +### Structure +- ✅ Created 5-file progressive disclosure system +- ✅ Added navigation table in README +- ✅ Added decision tree for routing +- ✅ Added "Reading Order by Task" guide +- ✅ Cross-referenced files throughout + +### New Content Added +- ✅ Zaraz Context (system/client properties) +- ✅ History Change trigger for SPA tracking +- ✅ Context Enrichers pattern +- ✅ Worker Variables pattern +- ✅ Consent management deep dive +- ✅ Tool-specific quirks (GA4, Facebook, Google Ads) +- ✅ GTM migration guide +- ✅ Comprehensive troubleshooting +- ✅ "When NOT to use Zaraz" section +- ✅ TypeScript type definitions + +### Preserved Content +- ✅ All original API methods +- ✅ E-commerce tracking examples +- ✅ Consent management +- ✅ Workers integration (expanded) +- ✅ Common patterns (expanded) +- ✅ Debugging tools +- ✅ Reference links + +## Progressive Disclosure Impact + +### Before (Monolithic) +All tasks loaded 366 lines regardless of need. + +### After (Progressive) +- **Track event task**: README (111) + api.md (287) = 398 lines +- **Debug issue**: gotchas.md (317) = 317 lines (13% reduction) +- **Configure tool**: configuration.md (307) = 307 lines (16% reduction) +- **SPA tracking**: README + patterns.md (SPA section) ~180 lines (51% reduction) + +**Net effect:** Task-specific loading reduces unnecessary content by 13-51% depending on use case. + +## File Summary + +### README.md (111 lines) +- Overview and core concepts +- Quick start guide +- When to use Zaraz vs Workers +- Navigation table +- Reading order by task +- Decision tree + +### api.md (287 lines) +- zaraz.track() +- zaraz.set() +- zaraz.ecommerce() +- Zaraz Context (system/client properties) +- zaraz.consent API +- zaraz.debug +- Cookie methods +- TypeScript definitions + +### configuration.md (307 lines) +- Dashboard setup flow +- Trigger types (including History Change) +- Tool configuration (GA4, Facebook, Google Ads) +- Actions and action rules +- Selective loading +- Consent management setup +- Privacy features +- Testing workflow + +### patterns.md (430 lines) +- SPA tracking (React, Vue, Next.js) +- User identification flows +- Complete e-commerce funnel +- A/B testing +- Worker integration (Context Enrichers, Worker Variables, HTML injection) +- Multi-tool coordination +- GTM migration +- Best practices + +### gotchas.md (317 lines) +- Events not firing (5-step debug process) +- Consent issues +- SPA tracking pitfalls +- Performance issues +- Tool-specific quirks +- Data layer issues +- Limits table +- When NOT to use Zaraz +- Debug checklist + +## Quality Metrics + +- ✅ All files use consistent markdown formatting +- ✅ Code examples include language tags +- ✅ Tables for structured data (limits, parameters, comparisons) +- ✅ Problem → Cause → Solution format in gotchas +- ✅ Cross-references between files +- ✅ No "see documentation" placeholders +- ✅ Real, actionable examples throughout +- ✅ Verified API syntax for Workers + +## Original Backup + +Original SKILL.md preserved as `_SKILL_old.md` for reference. diff --git a/plugins/cloudflare/skills/cloudflare/references/zaraz/README.md b/plugins/cloudflare/skills/cloudflare/references/zaraz/README.md new file mode 100644 index 0000000..0e28155 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/zaraz/README.md @@ -0,0 +1,111 @@ +# Cloudflare Zaraz + +Expert guidance for Cloudflare Zaraz - server-side tag manager for loading third-party tools at the edge. + +## What is Zaraz? + +Zaraz offloads third-party scripts (analytics, ads, chat, marketing) to Cloudflare's edge, improving site speed, privacy, and security. Zero client-side performance impact. + +**Core Concepts:** +- **Server-side execution** - Scripts run on Cloudflare, not user's browser +- **Single HTTP request** - All tools loaded via one endpoint +- **Privacy-first** - Control data sent to third parties +- **No client-side JS overhead** - Minimal browser impact + +## Quick Start + +1. Navigate to domain > Zaraz in Cloudflare dashboard +2. Click "Start setup" +3. Add tools (Google Analytics, Facebook Pixel, etc.) +4. Configure triggers (when tools fire) +5. Add tracking code to your site: + +```javascript +// Track page view +zaraz.track('page_view'); + +// Track custom event +zaraz.track('button_click', { button_id: 'cta' }); + +// Set user properties +zaraz.set('userId', 'user_123'); +``` + +## When to Use Zaraz + +**Use Zaraz when:** +- Adding multiple third-party tools (analytics, ads, marketing) +- Site performance is critical (no client-side JS overhead) +- Privacy compliance required (GDPR, CCPA) +- Non-technical teams need to manage tools + +**Use Workers directly when:** +- Building custom server-side tracking logic +- Need full control over data processing +- Integrating with complex backend systems +- Zaraz's tool library doesn't meet needs + +## In This Reference + +| File | Purpose | When to Read | +|------|---------|--------------| +| [api.md](./api.md) | Web API, zaraz object, consent methods | Implementing tracking calls | +| [configuration.md](./configuration.md) | Dashboard setup, triggers, tools | Initial setup, adding tools | +| [patterns.md](./patterns.md) | SPA, e-commerce, Worker integration | Best practices, common scenarios | +| [gotchas.md](./gotchas.md) | Troubleshooting, limits, pitfalls | Debugging issues | + +## Reading Order by Task + +| Task | Files to Read | +|------|---------------| +| Add analytics to site | README → configuration.md | +| Track custom events | README → api.md | +| Debug tracking issues | gotchas.md | +| SPA tracking | api.md → patterns.md (SPA section) | +| E-commerce tracking | api.md#ecommerce → patterns.md#ecommerce | +| Worker integration | patterns.md#worker-integration | +| GDPR compliance | api.md#consent → configuration.md#consent | + +## Decision Tree + +``` +What do you need? + +├─ Track events in browser → api.md +│ ├─ Page views, clicks → zaraz.track() +│ ├─ User properties → zaraz.set() +│ └─ E-commerce → zaraz.ecommerce() +│ +├─ Configure Zaraz → configuration.md +│ ├─ Add GA4/Facebook → tools setup +│ ├─ When tools fire → triggers +│ └─ GDPR consent → consent purposes +│ +├─ Integrate with Workers → patterns.md#worker-integration +│ ├─ Enrich context → Context Enrichers +│ └─ Inject tracking → HTML rewriting +│ +└─ Debug issues → gotchas.md + ├─ Events not firing → troubleshooting + ├─ Consent issues → consent debugging + └─ Performance → debugging tools +``` + +## Key Features + +- **100+ Pre-built Tools** - GA4, Facebook, Google Ads, TikTok, etc. +- **Zero Client Impact** - Runs at Cloudflare's edge, not browser +- **Privacy Controls** - Consent management, data filtering +- **Custom Tools** - Build Managed Components for proprietary systems +- **Worker Integration** - Enrich context, compute dynamic values +- **Debug Mode** - Real-time event inspection + +## Reference + +- [Zaraz Docs](https://developers.cloudflare.com/zaraz/) +- [Web API](https://developers.cloudflare.com/zaraz/web-api/) +- [Managed Components](https://developers.cloudflare.com/zaraz/advanced/load-custom-managed-component/) + +--- + +This skill focuses exclusively on Zaraz. For Workers development, see `cloudflare-workers` skill. diff --git a/plugins/cloudflare/skills/cloudflare/references/zaraz/api.md b/plugins/cloudflare/skills/cloudflare/references/zaraz/api.md new file mode 100644 index 0000000..5d8e1cc --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/zaraz/api.md @@ -0,0 +1,112 @@ +# Zaraz Web API + +Client-side JavaScript API for tracking events, setting properties, and managing consent. + +## zaraz.track() + +```javascript +zaraz.track('button_click'); +zaraz.track('purchase', { value: 99.99, currency: 'USD', item_id: '12345' }); +zaraz.track('pageview', { page_path: '/products', page_title: 'Products' }); // SPA +``` + +**Params:** `eventName` (string), `properties` (object, optional). Fire-and-forget. + +## zaraz.set() + +```javascript +zaraz.set('userId', 'user_12345'); +zaraz.set({ email: '[email protected]', plan: 'premium', country: 'US' }); +``` + +Properties persist for page session. Use for user identification and segmentation. + +## zaraz.ecommerce() + +```javascript +zaraz.ecommerce('Product Viewed', { product_id: 'SKU123', name: 'Widget', price: 49.99 }); +zaraz.ecommerce('Product Added', { product_id: 'SKU123', quantity: 2, price: 49.99 }); +zaraz.ecommerce('Order Completed', { + order_id: 'ORD-789', total: 149.98, currency: 'USD', + products: [{ product_id: 'SKU123', quantity: 2, price: 49.99 }] +}); +``` + +**Events:** `Product Viewed`, `Product Added`, `Product Removed`, `Cart Viewed`, `Checkout Started`, `Order Completed` + +Tools auto-map to GA4, Facebook CAPI, etc. + +## System Properties (Triggers) + +``` +{{system.page.url}} {{system.page.title}} {{system.page.referrer}} +{{system.device.ip}} {{system.device.userAgent}} {{system.device.language}} +{{system.cookies.name}} {{client.__zarazTrack.userId}} +``` + +## zaraz.consent + +```javascript +// Check +const purposes = zaraz.consent.getAll(); // { analytics: true, marketing: false } + +// Set +zaraz.consent.modal = true; // Show modal +zaraz.consent.setAll({ analytics: true, marketing: false }); +zaraz.consent.set('marketing', true); + +// Listen +zaraz.consent.addEventListener('consentChanged', () => { + if (zaraz.consent.getAll().marketing) zaraz.track('marketing_consent_granted'); +}); +``` + +**Flow:** Configure purposes in dashboard → Map tools to purposes → Show modal/set programmatically → Tools fire when allowed + +## zaraz.debug + +```javascript +zaraz.debug = true; +zaraz.track('test_event'); +console.log(zaraz.tools); // View loaded tools +``` + +## Cookie Methods + +```javascript +zaraz.getCookie('session_id'); // Zaraz namespace +zaraz.readCookie('_ga'); // Any cookie +``` + +## Async Behavior + +All methods fire-and-forget. Events batched and sent asynchronously: + +```javascript +zaraz.track('event1'); +zaraz.set('prop', 'value'); +zaraz.track('event2'); // All batched +``` + +## TypeScript Types + +```typescript +interface Zaraz { + track(event: string, properties?: Record): void; + set(key: string, value: unknown): void; + set(properties: Record): void; + ecommerce(event: string, properties: Record): void; + consent: { + getAll(): Record; + setAll(purposes: Record): void; + set(purpose: string, value: boolean): void; + addEventListener(event: 'consentChanged', callback: () => void): void; + modal: boolean; + }; + debug: boolean; + tools?: string[]; + getCookie(name: string): string | undefined; + readCookie(name: string): string | undefined; +} +declare global { interface Window { zaraz: Zaraz; } } +``` diff --git a/plugins/cloudflare/skills/cloudflare/references/zaraz/configuration.md b/plugins/cloudflare/skills/cloudflare/references/zaraz/configuration.md new file mode 100644 index 0000000..e2e534c --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/zaraz/configuration.md @@ -0,0 +1,90 @@ +# Zaraz Configuration + +## Dashboard Setup + +1. Domain → Zaraz → Start setup +2. Add tool (e.g., Google Analytics 4) +3. Enter credentials (GA4: `G-XXXXXXXXXX`) +4. Configure triggers +5. Save and Publish + +## Triggers + +| Type | When | Use Case | +|------|------|----------| +| Pageview | Page load | Track page views | +| Click | Element clicked | Button tracking | +| Form Submission | Form submitted | Lead capture | +| History Change | URL changes (SPA) | React/Vue routing | +| Variable Match | Custom condition | Conditional firing | + +### History Change (SPA) + +``` +Type: History Change +Event: pageview +``` + +Fires on `pushState`, `replaceState`, hash changes. **No manual tracking needed.** + +### Click Trigger + +``` +Type: Click +CSS Selector: .buy-button +Event: purchase_intent +Properties: + button_text: {{system.clickElement.text}} +``` + +## Tool Configuration + +**GA4:** +``` +Measurement ID: G-XXXXXXXXXX +Events: page_view, purchase, user_engagement +``` + +**Facebook Pixel:** +``` +Pixel ID: 1234567890123456 +Events: PageView, Purchase, AddToCart +``` + +**Google Ads:** +``` +Conversion ID: AW-XXXXXXXXX +Conversion Label: YYYYYYYYYY +``` + +## Consent Management + +1. Settings → Consent → Create purposes (analytics, marketing) +2. Map tools to purposes +3. Set behavior: "Do not load until consent granted" + +**Programmatic consent:** +```javascript +zaraz.consent.setAll({ analytics: true, marketing: true }); +``` + +## Privacy Features + +| Feature | Default | +|---------|---------| +| IP Anonymization | Enabled | +| Cookie Control | Via consent purposes | +| GDPR/CCPA | Consent modal | + +## Testing + +1. **Preview Mode** - test without publishing +2. **Debug Mode** - `zaraz.debug = true` +3. **Network tab** - filter "zaraz" + +## Limits + +| Resource | Limit | +|----------|-------| +| Event properties | 100KB | +| Consent purposes | 20 | diff --git a/plugins/cloudflare/skills/cloudflare/references/zaraz/gotchas.md b/plugins/cloudflare/skills/cloudflare/references/zaraz/gotchas.md new file mode 100644 index 0000000..eaa6b49 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/zaraz/gotchas.md @@ -0,0 +1,81 @@ +# Zaraz Gotchas + +## Events Not Firing + +**Check:** +1. Tool enabled in dashboard (green dot) +2. Trigger conditions met +3. Consent granted for tool's purpose +4. Tool credentials correct (GA4: `G-XXXXXXXXXX`, FB: numeric only) + +**Debug:** +```javascript +zaraz.debug = true; +console.log('Tools:', zaraz.tools); +console.log('Consent:', zaraz.consent.getAll()); +``` + +## Consent Issues + +**Modal not showing:** +```javascript +// Clear consent cookie +document.cookie = 'zaraz-consent=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;'; +location.reload(); +``` + +**Tools firing before consent:** Map tool to consent purpose with "Do not load until consent granted". + +## SPA Tracking + +**Route changes not tracked:** +1. Configure History Change trigger in dashboard +2. Hash routing (`#/path`) requires manual tracking: +```javascript +window.addEventListener('hashchange', () => { + zaraz.track('pageview', { page_path: location.pathname + location.hash }); +}); +``` + +**React fix:** +```javascript +const location = useLocation(); +useEffect(() => { + zaraz.track('pageview', { page_path: location.pathname }); +}, [location]); // Include dependency +``` + +## Performance + +**Slow page load:** +- Audit tool count (50+ degrades performance) +- Disable blocking triggers unless required +- Reduce event payload size (<100KB) + +## Tool-Specific Issues + +| Tool | Issue | Fix | +|------|-------|-----| +| GA4 | Events not in real-time | Wait 5-10 min, use DebugView | +| Facebook | Invalid Pixel ID | Use numeric only (no `fbpx_` prefix) | +| Google Ads | Conversions not attributed | Include `send_to: 'AW-XXX/LABEL'` | + +## Data Layer + +- Properties persist per page only - set on each page load +- Nested access: `{{client.__zarazTrack.user.plan}}` + +## Limits + +| Resource | Limit | +|----------|-------| +| Request size | 100KB | +| Consent purposes | 20 | +| API rate | 1000 req/sec | + +## When NOT to Use Zaraz + +- Server-to-server tracking (use Workers) +- Real-time bidirectional communication +- Binary data transmission +- Authentication flows diff --git a/plugins/cloudflare/skills/cloudflare/references/zaraz/patterns.md b/plugins/cloudflare/skills/cloudflare/references/zaraz/patterns.md new file mode 100644 index 0000000..c5ef967 --- /dev/null +++ b/plugins/cloudflare/skills/cloudflare/references/zaraz/patterns.md @@ -0,0 +1,74 @@ +# Zaraz Patterns + +## SPA Tracking + +**History Change Trigger (Recommended):** Configure in dashboard - no code needed, Zaraz auto-detects route changes. + +**Manual tracking (React/Vue/Next.js):** +```javascript +// On route change +zaraz.track('pageview', { page_path: pathname, page_title: document.title }); +``` + +## User Identification + +```javascript +// Login +zaraz.set({ userId: user.id, email: user.email, plan: user.plan }); +zaraz.track('login', { method: 'oauth' }); + +// Logout - set to null (cannot clear) +zaraz.set('userId', null); +``` + +## E-commerce Funnel + +| Event | Method | +|-------|--------| +| View | `zaraz.ecommerce('Product Viewed', { product_id, name, price })` | +| Add to cart | `zaraz.ecommerce('Product Added', { product_id, quantity })` | +| Checkout | `zaraz.ecommerce('Checkout Started', { cart_id, products: [...] })` | +| Purchase | `zaraz.ecommerce('Order Completed', { order_id, total, products })` | + +## A/B Testing + +```javascript +zaraz.set('experiment_checkout', variant); +zaraz.track('experiment_viewed', { experiment_id: 'checkout', variant }); +// On conversion +zaraz.track('experiment_conversion', { experiment_id, variant, value }); +``` + +## Worker Integration + +**Context Enricher** - Modify context before tools execute: +```typescript +export default { + async fetch(request, env) { + const body = await request.json(); + body.system.userRegion = request.cf?.region; + return Response.json(body); + } +}; +``` +Configure: Zaraz > Settings > Context Enrichers + +**Worker Variables** - Compute dynamic values server-side, use as `{{worker.variable_name}}`. + +## GTM Migration + +| GTM | Zaraz | +|-----|-------| +| `dataLayer.push({event: 'purchase'})` | `zaraz.ecommerce('Order Completed', {...})` | +| `{{Page URL}}` | `{{system.page.url}}` | +| `{{Page Title}}` | `{{system.page.title}}` | +| Page View trigger | Pageview trigger | +| Click trigger | Click (selector: `*`) | + +## Best Practices + +1. Use dashboard triggers over inline code +2. Enable History Change for SPAs (no manual code) +3. Debug with `zaraz.debug = true` +4. Implement consent early (GDPR/CCPA) +5. Use Context Enrichers for sensitive/server data diff --git a/plugins/cloudflare/skills/wrangler/LICENSE.txt b/plugins/cloudflare/skills/wrangler/LICENSE.txt new file mode 100644 index 0000000..13e25df --- /dev/null +++ b/plugins/cloudflare/skills/wrangler/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of + any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don\'t include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/plugins/cloudflare/skills/wrangler/SKILL.md b/plugins/cloudflare/skills/wrangler/SKILL.md new file mode 100644 index 0000000..296fa21 --- /dev/null +++ b/plugins/cloudflare/skills/wrangler/SKILL.md @@ -0,0 +1,897 @@ +--- +name: wrangler +description: Cloudflare Workers CLI for deploying, developing, and managing Workers, KV, R2, D1, Vectorize, Hyperdrive, Workers AI, Containers, Queues, Workflows, Pipelines, and Secrets Store. Load before running wrangler commands to ensure correct syntax and best practices. Biases towards retrieval from Cloudflare docs over pre-trained knowledge. +--- + +# Wrangler CLI + +Your knowledge of Wrangler CLI flags, config fields, and subcommands may be outdated. **Prefer retrieval over pre-training** for any Wrangler task. + +## Retrieval Sources + +Fetch the **latest** information before writing or reviewing Wrangler commands and config. Do not rely on baked-in knowledge for CLI flags, config fields, or binding shapes. + +| Source | How to retrieve | Use for | +|--------|----------------|---------| +| Wrangler docs | `https://developers.cloudflare.com/workers/wrangler/` | CLI commands, flags, config reference | +| Wrangler config schema | `node_modules/wrangler/config-schema.json` | Config fields, binding shapes, allowed values | +| Cloudflare docs | Search tool or `https://developers.cloudflare.com/workers/` | API reference, compatibility dates/flags | + +## FIRST: Verify Wrangler Installation + +```bash +wrangler --version # Requires v4.x+ +``` + +If not installed: +```bash +npm install -D wrangler@latest +``` + +## Key Guidelines + +- **Use `wrangler.jsonc`**: Prefer JSON config over TOML. Newer features are JSON-only. +- **Set `compatibility_date`**: Use a recent date (within 30 days). Check https://developers.cloudflare.com/workers/configuration/compatibility-dates/ +- **Generate types after config changes**: Run `wrangler types` to update TypeScript bindings. +- **Local dev defaults to local storage**: Bindings use local simulation unless `remote: true`. +- **Validate config before deploy**: Run `wrangler check` to catch errors early. +- **Use environments for staging/prod**: Define `env.staging` and `env.production` in config. + +## Quick Start: New Worker + +```bash +# Initialize new project +npx wrangler init my-worker + +# Or with a framework +npx create-cloudflare@latest my-app +``` + +## Quick Reference: Core Commands + +| Task | Command | +|------|---------| +| Start local dev server | `wrangler dev` | +| Deploy to Cloudflare | `wrangler deploy` | +| Deploy dry run | `wrangler deploy --dry-run` | +| Generate TypeScript types | `wrangler types` | +| Validate configuration | `wrangler check` | +| View live logs | `wrangler tail` | +| Delete Worker | `wrangler delete` | +| Auth status | `wrangler whoami` | + +--- + +## Configuration (wrangler.jsonc) + +### Minimal Config + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2026-01-01" +} +``` + +### Full Config with Bindings + +```jsonc +{ + "$schema": "./node_modules/wrangler/config-schema.json", + "name": "my-worker", + "main": "src/index.ts", + "compatibility_date": "2026-01-01", + "compatibility_flags": ["nodejs_compat_v2"], + + // Environment variables + "vars": { + "ENVIRONMENT": "production" + }, + + // KV Namespace + "kv_namespaces": [ + { "binding": "KV", "id": "" } + ], + + // R2 Bucket + "r2_buckets": [ + { "binding": "BUCKET", "bucket_name": "my-bucket" } + ], + + // D1 Database + "d1_databases": [ + { "binding": "DB", "database_name": "my-db", "database_id": "" } + ], + + // Workers AI (always remote) + "ai": { "binding": "AI" }, + + // Vectorize + "vectorize": [ + { "binding": "VECTOR_INDEX", "index_name": "my-index" } + ], + + // Hyperdrive + "hyperdrive": [ + { "binding": "HYPERDRIVE", "id": "" } + ], + + // Durable Objects + "durable_objects": { + "bindings": [ + { "name": "COUNTER", "class_name": "Counter" } + ] + }, + + // Cron triggers + "triggers": { + "crons": ["0 * * * *"] + }, + + // Environments + "env": { + "staging": { + "name": "my-worker-staging", + "vars": { "ENVIRONMENT": "staging" } + } + } +} +``` + +### Generate Types from Config + +```bash +# Generate worker-configuration.d.ts +wrangler types + +# Custom output path +wrangler types ./src/env.d.ts + +# Check types are up to date (CI) +wrangler types --check +``` + +--- + +## Local Development + +### Start Dev Server + +```bash +# Local mode (default) - uses local storage simulation +wrangler dev + +# With specific environment +wrangler dev --env staging + +# Force local-only (disable remote bindings) +wrangler dev --local + +# Remote mode - runs on Cloudflare edge (legacy) +wrangler dev --remote + +# Custom port +wrangler dev --port 8787 + +# Live reload for HTML changes +wrangler dev --live-reload + +# Test scheduled/cron handlers +wrangler dev --test-scheduled +# Then visit: http://localhost:8787/__scheduled +``` + +### Remote Bindings for Local Dev + +Use `remote: true` in binding config to connect to real resources while running locally: + +```jsonc +{ + "r2_buckets": [ + { "binding": "BUCKET", "bucket_name": "my-bucket", "remote": true } + ], + "ai": { "binding": "AI", "remote": true }, + "vectorize": [ + { "binding": "INDEX", "index_name": "my-index", "remote": true } + ] +} +``` + +**Recommended remote bindings**: AI (required), Vectorize, Browser Rendering, mTLS, Images. + +### Local Secrets + +Create `.dev.vars` for local development secrets: + +``` +API_KEY=local-dev-key +DATABASE_URL=postgres://localhost:5432/dev +``` + +--- + +## Deployment + +### Deploy Worker + +```bash +# Deploy to production +wrangler deploy + +# Deploy specific environment +wrangler deploy --env staging + +# Dry run (validate without deploying) +wrangler deploy --dry-run + +# Keep dashboard-set variables +wrangler deploy --keep-vars + +# Minify code +wrangler deploy --minify +``` + +### Manage Secrets + +```bash +# Set secret interactively +wrangler secret put API_KEY + +# Set from stdin +echo "secret-value" | wrangler secret put API_KEY + +# List secrets +wrangler secret list + +# Delete secret +wrangler secret delete API_KEY + +# Bulk secrets from JSON file +wrangler secret bulk secrets.json +``` + +### Versions and Rollback + +```bash +# List recent versions +wrangler versions list + +# View specific version +wrangler versions view + +# Rollback to previous version +wrangler rollback + +# Rollback to specific version +wrangler rollback +``` + +--- + +## KV (Key-Value Store) + +### Manage Namespaces + +```bash +# Create namespace +wrangler kv namespace create MY_KV + +# List namespaces +wrangler kv namespace list + +# Delete namespace +wrangler kv namespace delete --namespace-id +``` + +### Manage Keys + +```bash +# Put value +wrangler kv key put --namespace-id "key" "value" + +# Put with expiration (seconds) +wrangler kv key put --namespace-id "key" "value" --expiration-ttl 3600 + +# Get value +wrangler kv key get --namespace-id "key" + +# List keys +wrangler kv key list --namespace-id + +# Delete key +wrangler kv key delete --namespace-id "key" + +# Bulk put from JSON +wrangler kv bulk put --namespace-id data.json +``` + +### Config Binding + +```jsonc +{ + "kv_namespaces": [ + { "binding": "CACHE", "id": "" } + ] +} +``` + +--- + +## R2 (Object Storage) + +### Manage Buckets + +```bash +# Create bucket +wrangler r2 bucket create my-bucket + +# Create with location hint +wrangler r2 bucket create my-bucket --location wnam + +# List buckets +wrangler r2 bucket list + +# Get bucket info +wrangler r2 bucket info my-bucket + +# Delete bucket +wrangler r2 bucket delete my-bucket +``` + +### Manage Objects + +```bash +# Upload object +wrangler r2 object put my-bucket/path/file.txt --file ./local-file.txt + +# Download object +wrangler r2 object get my-bucket/path/file.txt + +# Delete object +wrangler r2 object delete my-bucket/path/file.txt +``` + +### Config Binding + +```jsonc +{ + "r2_buckets": [ + { "binding": "ASSETS", "bucket_name": "my-bucket" } + ] +} +``` + +--- + +## D1 (SQL Database) + +### Manage Databases + +```bash +# Create database +wrangler d1 create my-database + +# Create with location +wrangler d1 create my-database --location wnam + +# List databases +wrangler d1 list + +# Get database info +wrangler d1 info my-database + +# Delete database +wrangler d1 delete my-database +``` + +### Execute SQL + +```bash +# Execute SQL command (remote) +wrangler d1 execute my-database --remote --command "SELECT * FROM users" + +# Execute SQL file (remote) +wrangler d1 execute my-database --remote --file ./schema.sql + +# Execute locally +wrangler d1 execute my-database --local --command "SELECT * FROM users" +``` + +### Migrations + +```bash +# Create migration +wrangler d1 migrations create my-database create_users_table + +# List pending migrations +wrangler d1 migrations list my-database --local + +# Apply migrations locally +wrangler d1 migrations apply my-database --local + +# Apply migrations to remote +wrangler d1 migrations apply my-database --remote +``` + +### Export/Backup + +```bash +# Export schema and data +wrangler d1 export my-database --remote --output backup.sql + +# Export schema only +wrangler d1 export my-database --remote --output schema.sql --no-data +``` + +### Config Binding + +```jsonc +{ + "d1_databases": [ + { + "binding": "DB", + "database_name": "my-database", + "database_id": "", + "migrations_dir": "./migrations" + } + ] +} +``` + +--- + +## Vectorize (Vector Database) + +### Manage Indexes + +```bash +# Create index with dimensions +wrangler vectorize create my-index --dimensions 768 --metric cosine + +# Create with preset (auto-configures dimensions/metric) +wrangler vectorize create my-index --preset @cf/baai/bge-base-en-v1.5 + +# List indexes +wrangler vectorize list + +# Get index info +wrangler vectorize get my-index + +# Delete index +wrangler vectorize delete my-index +``` + +### Manage Vectors + +```bash +# Insert vectors from NDJSON file +wrangler vectorize insert my-index --file vectors.ndjson + +# Query vectors +wrangler vectorize query my-index --vector "[0.1, 0.2, ...]" --top-k 10 +``` + +### Config Binding + +```jsonc +{ + "vectorize": [ + { "binding": "SEARCH_INDEX", "index_name": "my-index" } + ] +} +``` + +--- + +## Hyperdrive (Database Accelerator) + +### Manage Configs + +```bash +# Create config +wrangler hyperdrive create my-hyperdrive \ + --connection-string "postgres://user:pass@host:5432/database" + +# List configs +wrangler hyperdrive list + +# Get config details +wrangler hyperdrive get + +# Update config +wrangler hyperdrive update --origin-password "new-password" + +# Delete config +wrangler hyperdrive delete +``` + +### Config Binding + +```jsonc +{ + "compatibility_flags": ["nodejs_compat_v2"], + "hyperdrive": [ + { "binding": "HYPERDRIVE", "id": "" } + ] +} +``` + +--- + +## Workers AI + +### List Models + +```bash +# List available models +wrangler ai models + +# List finetunes +wrangler ai finetune list +``` + +### Config Binding + +```jsonc +{ + "ai": { "binding": "AI" } +} +``` + +**Note**: Workers AI always runs remotely and incurs usage charges even in local dev. + +--- + +## Queues + +### Manage Queues + +```bash +# Create queue +wrangler queues create my-queue + +# List queues +wrangler queues list + +# Delete queue +wrangler queues delete my-queue + +# Add consumer to queue +wrangler queues consumer add my-queue my-worker + +# Remove consumer +wrangler queues consumer remove my-queue my-worker +``` + +### Config Binding + +```jsonc +{ + "queues": { + "producers": [ + { "binding": "MY_QUEUE", "queue": "my-queue" } + ], + "consumers": [ + { + "queue": "my-queue", + "max_batch_size": 10, + "max_batch_timeout": 30 + } + ] + } +} +``` + +--- + +## Containers + +### Build and Push Images + +```bash +# Build container image +wrangler containers build -t my-app:latest . + +# Build and push in one command +wrangler containers build -t my-app:latest . --push + +# Push existing image to Cloudflare registry +wrangler containers push my-app:latest +``` + +### Manage Containers + +```bash +# List containers +wrangler containers list + +# Get container info +wrangler containers info + +# Delete container +wrangler containers delete +``` + +### Manage Images + +```bash +# List images in registry +wrangler containers images list + +# Delete image +wrangler containers images delete my-app:latest +``` + +### Manage External Registries + +```bash +# List configured registries +wrangler containers registries list + +# Configure external registry (e.g., ECR) +wrangler containers registries configure \ + --public-credential + +# Delete registry configuration +wrangler containers registries delete +``` + +--- + +## Workflows + +### Manage Workflows + +```bash +# List workflows +wrangler workflows list + +# Describe workflow +wrangler workflows describe my-workflow + +# Trigger workflow instance +wrangler workflows trigger my-workflow + +# Trigger with parameters +wrangler workflows trigger my-workflow --params '{"key": "value"}' + +# Delete workflow +wrangler workflows delete my-workflow +``` + +### Manage Workflow Instances + +```bash +# List instances +wrangler workflows instances list my-workflow + +# Describe instance +wrangler workflows instances describe my-workflow + +# Terminate instance +wrangler workflows instances terminate my-workflow +``` + +### Config Binding + +```jsonc +{ + "workflows": [ + { + "binding": "MY_WORKFLOW", + "name": "my-workflow", + "class_name": "MyWorkflow" + } + ] +} +``` + +--- + +## Pipelines + +### Manage Pipelines + +```bash +# Create pipeline +wrangler pipelines create my-pipeline --r2 my-bucket + +# List pipelines +wrangler pipelines list + +# Show pipeline details +wrangler pipelines show my-pipeline + +# Update pipeline +wrangler pipelines update my-pipeline --batch-max-mb 100 + +# Delete pipeline +wrangler pipelines delete my-pipeline +``` + +### Config Binding + +```jsonc +{ + "pipelines": [ + { "binding": "MY_PIPELINE", "pipeline": "my-pipeline" } + ] +} +``` + +--- + +## Secrets Store + +### Manage Stores + +```bash +# Create store +wrangler secrets-store store create my-store + +# List stores +wrangler secrets-store store list + +# Delete store +wrangler secrets-store store delete +``` + +### Manage Secrets in Store + +```bash +# Add secret to store +wrangler secrets-store secret put my-secret + +# List secrets in store +wrangler secrets-store secret list + +# Get secret +wrangler secrets-store secret get my-secret + +# Delete secret from store +wrangler secrets-store secret delete my-secret +``` + +### Config Binding + +```jsonc +{ + "secrets_store_secrets": [ + { + "binding": "MY_SECRET", + "store_id": "", + "secret_name": "my-secret" + } + ] +} +``` + +--- + +## Pages (Frontend Deployment) + +```bash +# Create Pages project +wrangler pages project create my-site + +# Deploy directory to Pages +wrangler pages deploy ./dist + +# Deploy with specific branch +wrangler pages deploy ./dist --branch main + +# List deployments +wrangler pages deployment list --project-name my-site +``` + +--- + +## Observability + +### Tail Logs + +```bash +# Stream live logs +wrangler tail + +# Tail specific Worker +wrangler tail my-worker + +# Filter by status +wrangler tail --status error + +# Filter by search term +wrangler tail --search "error" + +# JSON output +wrangler tail --format json +``` + +### Config Logging + +```jsonc +{ + "observability": { + "enabled": true, + "head_sampling_rate": 1 + } +} +``` + +--- + +## Testing + +### Local Testing with Vitest + +```bash +npm install -D @cloudflare/vitest-pool-workers vitest +``` + +`vitest.config.ts`: +```typescript +import { defineWorkersConfig } from "@cloudflare/vitest-pool-workers/config"; + +export default defineWorkersConfig({ + test: { + poolOptions: { + workers: { + wrangler: { configPath: "./wrangler.jsonc" }, + }, + }, + }, +}); +``` + +### Test Scheduled Events + +```bash +# Enable in dev +wrangler dev --test-scheduled + +# Trigger via HTTP +curl http://localhost:8787/__scheduled +``` + +--- + +## Troubleshooting + +### Common Issues + +| Issue | Solution | +|-------|----------| +| `command not found: wrangler` | Install: `npm install -D wrangler` | +| Auth errors | Run `wrangler login` | +| Config validation errors | Run `wrangler check` | +| Type errors after config change | Run `wrangler types` | +| Local storage not persisting | Check `.wrangler/state` directory | +| Binding undefined in Worker | Verify binding name matches config exactly | + +### Debug Commands + +```bash +# Check auth status +wrangler whoami + +# Validate config +wrangler check + +# View config schema +wrangler docs configuration +``` + +--- + +## Best Practices + +1. **Version control `wrangler.jsonc`**: Treat as source of truth for Worker config. +2. **Use automatic provisioning**: Omit resource IDs for auto-creation on deploy. +3. **Run `wrangler types` in CI**: Add to build step to catch binding mismatches. +4. **Use environments**: Separate staging/production with `env.staging`, `env.production`. +5. **Set `compatibility_date`**: Update quarterly to get new runtime features. +6. **Use `.dev.vars` for local secrets**: Never commit secrets to config. +7. **Test locally first**: `wrangler dev` with local bindings before deploying. +8. **Use `--dry-run` before major deploys**: Validate changes without deployment. diff --git a/plugins/cloudflare/skills/wrangler/agents/openai.yaml b/plugins/cloudflare/skills/wrangler/agents/openai.yaml new file mode 100644 index 0000000..c853269 --- /dev/null +++ b/plugins/cloudflare/skills/wrangler/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Wrangler" + short_description: "Use the Cloudflare Workers CLI safely" + icon_small: "./assets/cloudflare-small.svg" + icon_large: "./assets/cloudflare.png" + default_prompt: "Use Wrangler to set up, review, or fix this Cloudflare Worker config and deployment flow without guessing CLI flags." diff --git a/plugins/cloudflare/skills/wrangler/assets/cloudflare-small.svg b/plugins/cloudflare/skills/wrangler/assets/cloudflare-small.svg new file mode 100644 index 0000000..7feedbc --- /dev/null +++ b/plugins/cloudflare/skills/wrangler/assets/cloudflare-small.svg @@ -0,0 +1,3 @@ + diff --git a/plugins/cloudflare/skills/wrangler/assets/cloudflare.png b/plugins/cloudflare/skills/wrangler/assets/cloudflare.png new file mode 100644 index 0000000..cf23088 Binary files /dev/null and b/plugins/cloudflare/skills/wrangler/assets/cloudflare.png differ diff --git a/plugins/github/.app.json b/plugins/github/.app.json new file mode 100644 index 0000000..d218602 --- /dev/null +++ b/plugins/github/.app.json @@ -0,0 +1,7 @@ +{ + "apps": { + "github": { + "id": "github" + } + } +} diff --git a/plugins/github/.codex-plugin/plugin.json b/plugins/github/.codex-plugin/plugin.json new file mode 100644 index 0000000..87d108b --- /dev/null +++ b/plugins/github/.codex-plugin/plugin.json @@ -0,0 +1,41 @@ +{ + "name": "github", + "version": "0.1.0", + "description": "Triage GitHub work, review pull requests, and plan safe patches from Codex.", + "author": { + "name": "OpenAI", + "email": "support@openai.com", + "url": "https://openai.com/" + }, + "homepage": "https://github.com/", + "repository": "https://github.com/openai/plugins", + "license": "MIT", + "keywords": [ + "github", + "triage", + "pull-requests", + "reviews" + ], + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "apps": "./.app.json", + "interface": { + "displayName": "GitHub", + "shortDescription": "Triage issues and review pull requests", + "longDescription": "Use GitHub context to triage repository work, review pull requests, and prepare safe patch plans through the connected GitHub app and MCP server.", + "developerName": "OpenAI", + "category": "Productivity", + "capabilities": [ + "Interactive", + "Write" + ], + "websiteURL": "https://github.com/", + "privacyPolicyURL": "https://docs.github.com/en/site-policy/privacy-policies/github-general-privacy-statement", + "termsOfServiceURL": "https://docs.github.com/en/site-policy/github-terms/github-terms-of-service", + "defaultPrompt": "Use GitHub context to triage issues and pull requests for this repo, then recommend the top next actions with clear risks and testing notes.", + "composerIcon": "./assets/github-small.svg", + "logo": "./assets/github.svg", + "screenshots": [], + "brandColor": "#24292F" + } +} diff --git a/plugins/github/.mcp.json b/plugins/github/.mcp.json new file mode 100644 index 0000000..9e3c4b8 --- /dev/null +++ b/plugins/github/.mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "github": { + "type": "streamable_http", + "url": "https://api.githubcopilot.com/mcp/", + "bearer_token_env_var": "CODEX_GITHUB_PERSONAL_ACCESS_TOKEN" + } + } +} diff --git a/plugins/github/assets/github-small.svg b/plugins/github/assets/github-small.svg new file mode 100644 index 0000000..dad335d --- /dev/null +++ b/plugins/github/assets/github-small.svg @@ -0,0 +1,4 @@ + + + + diff --git a/plugins/github/assets/github.svg b/plugins/github/assets/github.svg new file mode 100644 index 0000000..3c9e578 --- /dev/null +++ b/plugins/github/assets/github.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/plugins/github/skills/repo-triage/SKILL.md b/plugins/github/skills/repo-triage/SKILL.md new file mode 100644 index 0000000..eb9c3d4 --- /dev/null +++ b/plugins/github/skills/repo-triage/SKILL.md @@ -0,0 +1,34 @@ +--- +name: repo-triage +description: Triage GitHub issues and pull requests, build a prioritized queue, and propose next actions with clear risk/testing notes. +metadata: + short-description: GitHub repo issue/PR triage +--- + +# Repo Triage Skill + +Use this skill when the task is to understand the state of a repository's issue or PR queue and produce an actionable triage outcome. + +## What this skill is for + +- Prioritizing incoming issues by impact, confidence, and unblockability +- Grouping PRs by review state and risk +- Producing a clean handoff summary for engineering leads or oncall rotations + +## Workflow + +1. Collect the requested issue/PR set using the GitHub MCP tools. +2. Normalize references and labels (see `references/query_patterns.md`). +3. Build a triage table with: status, owner, risk, blockers, and recommended next action. +4. Call out missing tests, missing repros, and stale branches explicitly. +5. End with the top 3 actions the user can take next. + +## Output conventions + +- Keep summaries concise and decision-oriented. +- Prefer explicit evidence (links, PR numbers, failing checks) over speculation. +- If the request is review-centric, delegate to the plugin's PR review sub-agent prompt. + +## Optional helper script + +If the user provides text blobs (issue lists, changelogs, pasted PR summaries), use `scripts/extract_refs.py` to normalize GitHub issue and PR references before triage. diff --git a/plugins/github/skills/repo-triage/agents/openai.yaml b/plugins/github/skills/repo-triage/agents/openai.yaml new file mode 100644 index 0000000..265583b --- /dev/null +++ b/plugins/github/skills/repo-triage/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Repo Triage" + short_description: "Prioritize GitHub issues and pull requests with concrete next actions" + default_prompt: "Triage the current GitHub issue/PR queue and produce a prioritized action list." diff --git a/plugins/github/skills/repo-triage/references/query_patterns.md b/plugins/github/skills/repo-triage/references/query_patterns.md new file mode 100644 index 0000000..94936a7 --- /dev/null +++ b/plugins/github/skills/repo-triage/references/query_patterns.md @@ -0,0 +1,25 @@ +# Query Patterns (Draft) + +Use these patterns when querying a GitHub MCP server for triage workflows. + +## Issues + +- `is:issue is:open sort:updated-desc` for fresh inbound triage +- `is:issue is:open label:bug no:assignee` for unowned bugs +- `is:issue is:open label:regression` for release risk scanning + +## Pull Requests + +- `is:pr is:open review:required` for review queue triage +- `is:pr is:open status:failure` for broken CI +- `is:pr is:open updated:<7d` for stale active work + +## Triage heuristics + +Capture at least these fields in your final output: + +- `owner` +- `current_state` +- `blocking_reason` +- `risk_level` +- `next_action` diff --git a/plugins/github/skills/repo-triage/scripts/extract_refs.py b/plugins/github/skills/repo-triage/scripts/extract_refs.py new file mode 100755 index 0000000..237a478 --- /dev/null +++ b/plugins/github/skills/repo-triage/scripts/extract_refs.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Extract GitHub issue/PR references from stdin. + +Utility for skill users who paste arbitrary text before triage. +""" + +from __future__ import annotations + +import re +import sys + +PATTERN = re.compile(r"(?:(?P[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+))?#(?P\d+)") + + +def main() -> int: + text = sys.stdin.read() + if not text: + print("No input provided", file=sys.stderr) + return 1 + + seen = set() + for match in PATTERN.finditer(text): + repo = match.group("repo") or "" + num = match.group("num") + key = (repo, num) + if key in seen: + continue + seen.add(key) + print(f"{repo}#{num}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/notion/.codex-plugin/plugin.json b/plugins/notion/.codex-plugin/plugin.json new file mode 100644 index 0000000..fe8db82 --- /dev/null +++ b/plugins/notion/.codex-plugin/plugin.json @@ -0,0 +1,39 @@ +{ + "name": "notion", + "version": "0.1.0", + "description": "Capture knowledge, research pages, and turn Notion specs into execution artifacts.", + "author": { + "name": "OpenAI", + "email": "support@openai.com", + "url": "https://openai.com/" + }, + "homepage": "https://www.notion.com/", + "repository": "https://github.com/openai/plugins", + "license": "MIT", + "keywords": [ + "notion", + "knowledge-management", + "documentation" + ], + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "interface": { + "displayName": "Notion", + "shortDescription": "Capture knowledge and turn specs into plans", + "longDescription": "Use Notion context to capture decisions, prepare meetings, research linked pages, and turn specs into implementation plans through the connected Notion MCP server.", + "developerName": "OpenAI", + "category": "Productivity", + "capabilities": [ + "Interactive", + "Write" + ], + "websiteURL": "https://www.notion.com/", + "privacyPolicyURL": "https://www.notion.com/trust/privacy-policy", + "termsOfServiceURL": "https://www.notion.so/terms", + "defaultPrompt": "Use Notion context to capture knowledge, research linked pages, or turn a spec into an implementation plan with connected tasks.", + "composerIcon": "./assets/notion-small.svg", + "logo": "./assets/notion.png", + "screenshots": [], + "brandColor": "#000000" + } +} diff --git a/plugins/notion/.mcp.json b/plugins/notion/.mcp.json new file mode 100644 index 0000000..ad33486 --- /dev/null +++ b/plugins/notion/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "notion": { + "type": "streamable_http", + "url": "https://mcp.notion.com/mcp" + } + } +} diff --git a/plugins/notion/assets/notion-small.svg b/plugins/notion/assets/notion-small.svg new file mode 100644 index 0000000..455d4be --- /dev/null +++ b/plugins/notion/assets/notion-small.svg @@ -0,0 +1,11 @@ + diff --git a/plugins/notion/assets/notion.png b/plugins/notion/assets/notion.png new file mode 100644 index 0000000..085023c Binary files /dev/null and b/plugins/notion/assets/notion.png differ diff --git a/plugins/notion/skills/notion-knowledge-capture/LICENSE.txt b/plugins/notion/skills/notion-knowledge-capture/LICENSE.txt new file mode 100644 index 0000000..0871708 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/LICENSE.txt @@ -0,0 +1,7 @@ +Copyright 2025 Notion Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/notion/skills/notion-knowledge-capture/SKILL.md b/plugins/notion/skills/notion-knowledge-capture/SKILL.md new file mode 100644 index 0000000..7b3a489 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/SKILL.md @@ -0,0 +1,56 @@ +--- +name: notion-knowledge-capture +description: Capture conversations and decisions into structured Notion pages; use when turning chats/notes into wiki entries, how-tos, decisions, or FAQs with proper linking. +metadata: + short-description: Capture conversations into structured Notion pages +--- + +# Knowledge Capture + +Convert conversations and notes into structured, linkable Notion pages for easy reuse. + +## Quick start +1) Clarify what to capture (decision, how-to, FAQ, learning, documentation) and target audience. +2) Identify the right database/template in `reference/` (team wiki, how-to, FAQ, decision log, learning, documentation). +3) Pull any prior context from Notion with `Notion:search` → `Notion:fetch` (existing pages to update/link). +4) Draft the page with `Notion:notion-create-pages` using the database’s schema; include summary, context, source links, and tags/owners. +5) Link from hub pages and related records; update status/owners with `Notion:notion-update-page` as the source evolves. + +## Workflow +### 0) If any MCP call fails because Notion MCP is not connected, pause and set it up: +1. Add the Notion MCP: + - `codex mcp add notion --url https://mcp.notion.com/mcp` +2. Enable remote MCP client: + - Set `[features].rmcp_client = true` in `config.toml` **or** run `codex --enable rmcp_client` +3. Log in with OAuth: + - `codex mcp login notion` + +After successful login, the user will have to restart codex. You should finish your answer and tell them so when they try again they can continue with Step 1. + +### 1) Define the capture +- Ask purpose, audience, freshness, and whether this is new or an update. +- Determine content type: decision, how-to, FAQ, concept/wiki entry, learning/note, documentation page. + +### 2) Locate destination +- Pick the correct database using `reference/*-database.md` guides; confirm required properties (title, tags, owner, status, date, relations). +- If multiple candidate databases, ask the user which to use; otherwise, create in the primary wiki/documentation DB. + +### 3) Extract and structure +- Extract facts, decisions, actions, and rationale from the conversation. +- For decisions, record alternatives, rationale, and outcomes. +- For how-tos/docs, capture steps, pre-reqs, links to assets/code, and edge cases. +- For FAQs, phrase as Q&A with concise answers and links to deeper docs. + +### 4) Create/update in Notion +- Use `Notion:notion-create-pages` with the correct `data_source_id`; set properties (title, tags, owner, status, dates, relations). +- Use templates in `reference/` to structure content (section headers, checklists). +- If updating an existing page, fetch then edit via `Notion:notion-update-page`. + +### 5) Link and surface +- Add relations/backlinks to hub pages, related specs/docs, and teams. +- Add a short summary/changelog for future readers. +- If follow-up tasks exist, create tasks in the relevant database and link them. + +## References and examples +- `reference/` — database schemas and templates (e.g., `team-wiki-database.md`, `how-to-guide-database.md`, `faq-database.md`, `decision-log-database.md`, `documentation-database.md`, `learning-database.md`, `database-best-practices.md`). +- `examples/` — capture patterns in practice (e.g., `decision-capture.md`, `how-to-guide.md`, `conversation-to-faq.md`). diff --git a/plugins/notion/skills/notion-knowledge-capture/agents/openai.yaml b/plugins/notion/skills/notion-knowledge-capture/agents/openai.yaml new file mode 100644 index 0000000..4bc9f28 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/agents/openai.yaml @@ -0,0 +1,14 @@ +interface: + display_name: "Notion Knowledge Capture" + short_description: "Capture conversations into structured Notion pages" + icon_small: "./assets/notion-small.svg" + icon_large: "./assets/notion.png" + default_prompt: "Capture this conversation into structured Notion pages with decisions, action items, and owners when known." + +dependencies: + tools: + - type: "mcp" + value: "notion" + description: "Notion MCP server" + transport: "streamable_http" + url: "https://mcp.notion.com/mcp" diff --git a/plugins/notion/skills/notion-knowledge-capture/assets/notion-small.svg b/plugins/notion/skills/notion-knowledge-capture/assets/notion-small.svg new file mode 100644 index 0000000..455d4be --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/assets/notion-small.svg @@ -0,0 +1,11 @@ + diff --git a/plugins/notion/skills/notion-knowledge-capture/assets/notion.png b/plugins/notion/skills/notion-knowledge-capture/assets/notion.png new file mode 100644 index 0000000..085023c Binary files /dev/null and b/plugins/notion/skills/notion-knowledge-capture/assets/notion.png differ diff --git a/plugins/notion/skills/notion-knowledge-capture/evaluations/README.md b/plugins/notion/skills/notion-knowledge-capture/evaluations/README.md new file mode 100644 index 0000000..e140f31 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/evaluations/README.md @@ -0,0 +1,95 @@ +# Knowledge Capture Skill Evaluations + +Evaluation scenarios for testing the Knowledge Capture skill across different Codex models. + +## Purpose + +These evaluations ensure the Knowledge Capture skill: +- Correctly identifies content types (how-to guides, FAQs, decision records, wikis) +- Extracts relevant information from conversations +- Structures content appropriately for each type +- Searches and places content in the right Notion location +- Works consistently across Haiku, Sonnet, and Opus + +## Evaluation Files + +### conversation-to-wiki.json +Tests capturing conversation content as a how-to guide for the team wiki. + +**Scenario**: Save deployment discussion to wiki +**Key Behaviors**: +- Extracts steps, gotchas, and best practices from conversation +- Identifies content as How-To Guide +- Structures with proper sections (Overview, Prerequisites, Steps, Troubleshooting) +- Searches for team wiki location +- Preserves technical details (commands, configs) + +### decision-record.json +Tests capturing architectural or technical decisions with full context. + +**Scenario**: Document database migration decision +**Key Behaviors**: +- Extracts decision context, alternatives, and rationale +- Follows decision record structure (Context, Decision, Alternatives, Consequences) +- Captures both selected and rejected options with reasoning +- Places in decision log or ADR database +- Links to related technical documentation + +## Running Evaluations + +1. Enable the `knowledge-capture` skill +2. Submit the query from the evaluation file +3. Provide conversation context as specified +4. Verify all expected behaviors are met +5. Check success criteria for quality +6. Test with Haiku, Sonnet, and Opus + +## Expected Skill Behaviors + +Knowledge Capture evaluations should verify: + +### Content Extraction +- Accurately captures key points from conversation context +- Preserves specific technical details, not generic placeholders +- Maintains context and nuance from discussion + +### Content Type Selection +- Correctly identifies appropriate content type (how-to, FAQ, decision record, wiki page) +- Uses matching structure from reference documentation +- Applies proper Notion markdown formatting + +### Notion Integration +- Searches for appropriate target location (wiki, decision log, etc.) +- Creates well-structured pages with clear titles +- Uses proper parent placement +- Includes discoverable titles and metadata + +### Quality Standards +- Content is actionable and future-reference ready +- Technical accuracy is preserved +- Organization aids discoverability +- Formatting enhances readability + +## Creating New Evaluations + +When adding Knowledge Capture evaluations: + +1. **Use realistic conversation content** - Include actual technical details, decisions, or processes +2. **Test different content types** - How-to guides, FAQs, decision records, meeting notes, learnings +3. **Vary complexity** - Simple captures vs. complex technical discussions +4. **Test discovery** - Finding the right wiki section or database +5. **Include edge cases** - Unclear content types, minimal context, overlapping categories + +## Example Success Criteria + +**Good** (specific, testable): +- "Structures content using How-To format with numbered steps" +- "Preserves exact bash commands from conversation" +- "Creates page with title format 'How to [Action]'" +- "Places in Engineering Wiki → Deployment section" + +**Bad** (vague, untestable): +- "Creates good documentation" +- "Uses appropriate structure" +- "Saves to the right place" + diff --git a/plugins/notion/skills/notion-knowledge-capture/evaluations/conversation-to-wiki.json b/plugins/notion/skills/notion-knowledge-capture/evaluations/conversation-to-wiki.json new file mode 100644 index 0000000..c96772a --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/evaluations/conversation-to-wiki.json @@ -0,0 +1,31 @@ +{ + "name": "Save Conversation to Wiki", + "skills": ["knowledge-capture"], + "query": "Save this conversation about deploying our application to production to the team wiki", + "context": "Preceding conversation contains discussion about deployment process, including steps, gotchas, and best practices", + "expected_behavior": [ + "Extracts key information from conversation context (deployment steps, gotchas, best practices)", + "Identifies content type as How-To Guide based on procedural nature", + "Structures content using How-To structure: Overview → Prerequisites → Steps (numbered) → Verification → Troubleshooting → Related", + "Organizes information into clear sections with proper headings", + "Includes specific commands, configurations, or examples from conversation", + "Adds context about why/when to use this process in Overview section", + "Notes common issues and solutions mentioned in discussion in Troubleshooting section", + "Uses Notion:search to find team wiki location or asks user", + "Creates page using Notion:notion-create-pages with structured content and appropriate parent", + "Uses clear, descriptive title like 'How to Deploy to Production'", + "Applies Notion markdown formatting (headings, code blocks, bullets)", + "Suggests tags/categories for discoverability if wiki database" + ], + "success_criteria": [ + "Content is structured using How-To format from SKILL.md content types", + "Key points from conversation are captured accurately (not generic)", + "Information is organized with proper Notion markdown (##, ###, bullets, code blocks)", + "Specific technical details (commands, configs) are preserved from conversation", + "Document is written for future reference with clear step-by-step instructions", + "Title is searchable and descriptive (e.g., 'How to Deploy to Production')", + "Page is placed in appropriate wiki location (general wiki or specific section)", + "Uses correct tool name (Notion:notion-create-pages)" + ] +} + diff --git a/plugins/notion/skills/notion-knowledge-capture/evaluations/decision-record.json b/plugins/notion/skills/notion-knowledge-capture/evaluations/decision-record.json new file mode 100644 index 0000000..51f935d --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/evaluations/decision-record.json @@ -0,0 +1,31 @@ +{ + "name": "Create Decision Record", + "skills": ["knowledge-capture"], + "query": "Document our decision to use PostgreSQL instead of MongoDB for our new service", + "context": "User has just explained the decision with rationale, options considered, and trade-offs", + "expected_behavior": [ + "Recognizes this as a decision record (architectural decision) from conversation context", + "Uses Decision structure: Context → Decision → Rationale → Options Considered (with Pros/Cons) → Consequences → Implementation", + "Extracts from context: decision made, options considered (PostgreSQL vs MongoDB), rationale, trade-offs", + "Creates document with proper structure including Date, Status (Accepted), and Deciders", + "Includes both positive and negative consequences (trade-offs) in Consequences section", + "Uses Notion:search to check if decision log database exists", + "If database exists, asks whether to add there or create standalone page", + "If creating in database, fetches schema using Notion:fetch and sets properties: Decision title, Date, Status, Domain (Architecture), Deciders, Impact", + "Uses Notion:notion-create-pages with parent: { data_source_id } for database or { page_id } for parent page", + "Applies proper Notion markdown formatting with sections", + "Suggests linking from architecture docs or project pages" + ], + "success_criteria": [ + "Document follows Decision structure from SKILL.md content types", + "All key sections present: Context, Decision, Rationale, Options Considered (with Pros/Cons for each), Consequences, Implementation", + "Decision is clearly stated (PostgreSQL chosen over MongoDB)", + "Options that were considered are documented with pros/cons structure", + "Rationale explains why PostgreSQL was chosen based on conversation context", + "Consequences include both positive (benefits) and negative (trade-offs)", + "If in database, properties are set correctly from schema (Decision, Date, Status: Accepted, Domain: Architecture, Impact)", + "Document is dated and has status 'Accepted'", + "Uses correct tool names (Notion:search, Notion:fetch, Notion:notion-create-pages)" + ] +} + diff --git a/plugins/notion/skills/notion-knowledge-capture/examples/conversation-to-faq.md b/plugins/notion/skills/notion-knowledge-capture/examples/conversation-to-faq.md new file mode 100644 index 0000000..c4944b8 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/examples/conversation-to-faq.md @@ -0,0 +1,226 @@ +# Example: Conversation to FAQ + +## User Request + +> "Save this conversation about deployment troubleshooting to the FAQ" + +**Context**: User just had a conversation explaining how to troubleshoot common deployment errors. + +## Conversation Summary + +The conversation covered: +1. **Question**: "Why does deployment fail with 'port already in use' error?" +2. **Answer**: Process from previous deployment still running, need to kill it +3. **Question**: "How do I find and kill the process?" +4. **Commands shared**: `lsof -ti:3000 | xargs kill -9` or use `pm2 restart app` +5. **Question**: "What about 'cannot connect to database' errors?" +6. **Answer**: Check database credentials, verify database is running, check network connectivity +7. **Best practice**: Always check logs first (`pm2 logs` or `docker logs`) + +## Workflow Execution + +### Step 1: Identify Content Type + +This is Q&A content → **FAQ Entry** format is appropriate + +### Step 2: Structure FAQ Entries + +Extract distinct Q&A pairs from conversation: + +**FAQ 1**: Port already in use error +**FAQ 2**: Cannot connect to database error +**FAQ 3**: General troubleshooting approach + +### Step 3: Determine Destination + +``` +Notion:search +query: "FAQ deployment" +query_type: "internal" +``` + +Found: "Deployment FAQ" database in Engineering Docs + +``` +Notion:fetch +id: "deployment-faq-database-id" +``` + +**Schema**: +- Question (title) +- Category (select): Deployment, Configuration, Troubleshooting, etc. +- Tags (multi_select) +- Last Reviewed (date) + +### Step 4: Create FAQ Entries + +**FAQ Entry 1: Port Already in Use** + +``` +Notion:notion-create-pages + +parent: { data_source_id: "collection://faq-db-uuid" } + +pages: [{ + properties: { + "Question": "Why does deployment fail with 'port already in use' error?", + "Category": "Troubleshooting", + "Tags": "deployment, errors, ports", + "date:Last Reviewed:start": "2025-10-14", + "date:Last Reviewed:is_datetime": 0 + }, + content: "## Short Answer\n\nThe deployment port (usually 3000) is still occupied by a process from a previous deployment. You need to kill the existing process before deploying again.\n\n## Detailed Explanation\n\nWhen you deploy the application, it tries to bind to a specific port (e.g., port 3000). If a previous deployment didn't shut down cleanly, that process may still be running and holding the port. This prevents the new deployment from starting.\n\n**Common causes**:\n- Previous deployment crashed without cleanup\n- Manual `node` process started and forgotten\n- PM2 or other process manager didn't restart properly\n- Multiple deployments attempted simultaneously\n\n## Solution\n\n### Option 1: Kill the process using the port\n\n```bash\n# Find and kill process on port 3000\nlsof -ti:3000 | xargs kill -9\n```\n\nReplace `3000` with your application's port number.\n\n### Option 2: If using PM2\n\n```bash\n# Restart the application\npm2 restart app\n\n# Or delete and restart\npm2 delete app\npm2 start app.js --name app\n```\n\n### Option 3: Check all node processes\n\n```bash\n# List all node processes\nps aux | grep node\n\n# Kill specific process by PID\nkill -9 \n```\n\n## Prevention\n\n1. **Use process managers**: PM2, systemd, or Docker handle cleanup automatically\n2. **Graceful shutdown**: Implement proper shutdown handlers in your app\n3. **Health checks**: Monitor if previous deployment shut down before starting new one\n\n```javascript\n// Graceful shutdown example\nprocess.on('SIGTERM', () => {\n server.close(() => {\n console.log('Server closed');\n process.exit(0);\n });\n});\n```\n\n## Verification\n\nAfter killing the process, verify the port is free:\n\n```bash\nlsof -ti:3000\n# Should return nothing if port is free\n```\n\n## Related Questions\n- How do I check what's using a port?\n- How do I configure the application port?\n- PM2 deployment best practices\n\n## Last Updated\nOctober 14, 2025" +}] +``` + +**FAQ Entry 2: Database Connection Errors** + +``` +Notion:notion-create-pages + +parent: { data_source_id: "collection://faq-db-uuid" } + +pages: [{ + properties: { + "Question": "Why do I get 'cannot connect to database' errors during deployment?", + "Category": "Troubleshooting", + "Tags": "deployment, database, errors", + "date:Last Reviewed:start": "2025-10-14", + "date:Last Reviewed:is_datetime": 0 + }, + content: "## Short Answer\n\nDatabase connection errors usually mean either the database isn't running, credentials are incorrect, or there's a network connectivity issue. Check database status, verify credentials, and test connectivity.\n\n## Detailed Explanation\n\nThe application can't establish a connection to the database during startup. This prevents the application from initializing properly.\n\n**Common causes**:\n- Database service isn't running\n- Incorrect connection credentials\n- Network connectivity issues (firewall, security groups)\n- Database host/port misconfigured\n- Database is at connection limit\n- SSL/TLS configuration mismatch\n\n## Troubleshooting Steps\n\n### Step 1: Check database status\n\n```bash\n# For local PostgreSQL\npg_isready -h localhost -p 5432\n\n# For Docker\ndocker ps | grep postgres\n\n# For MongoDB\nmongosh --eval \"db.adminCommand('ping')\"\n```\n\n### Step 2: Verify credentials\n\nCheck your `.env` or configuration file:\n\n```bash\n# Common environment variables\nDB_HOST=localhost\nDB_PORT=5432\nDB_NAME=myapp_production\nDB_USER=myapp_user\nDB_PASSWORD=***********\n```\n\nTest connection manually:\n\n```bash\n# PostgreSQL\npsql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME\n\n# MongoDB\nmongosh \"mongodb://$DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_NAME\"\n```\n\n### Step 3: Check network connectivity\n\n```bash\n# Test if port is reachable\ntelnet $DB_HOST $DB_PORT\n\n# Or using nc\nnc -zv $DB_HOST $DB_PORT\n\n# Check firewall rules (if applicable)\nsudo iptables -L\n```\n\n### Step 4: Check application logs\n\n```bash\n# PM2 logs\npm2 logs app\n\n# Docker logs\ndocker logs container-name\n\n# Application logs\ntail -f /var/log/app/error.log\n```\n\nLook for specific error messages:\n- `ECONNREFUSED`: Database not running or wrong host/port\n- `Authentication failed`: Wrong credentials\n- `Timeout`: Network/firewall issue\n- `Too many connections`: Database connection limit reached\n\n## Solutions by Error Type\n\n### Database Not Running\n\n```bash\n# Start PostgreSQL\nsudo systemctl start postgresql\n\n# Start via Docker\ndocker start postgres-container\n```\n\n### Wrong Credentials\n\n1. Reset database password\n2. Update `.env` file\n3. Restart application\n\n### Connection Limit Reached\n\n```sql\n-- Check current connections (PostgreSQL)\nSELECT count(*) FROM pg_stat_activity;\n\n-- Increase max connections\nALTER SYSTEM SET max_connections = 200;\n```\n\n### SSL/TLS Issues\n\nAdd to connection string:\n```\nssl=true&sslmode=require\n```\n\nOr disable SSL for dev:\n```\nsslmode=disable\n```\n\n## Prevention\n\n1. **Connection pooling**: Limits concurrent connections\n```javascript\npool: {\n min: 2,\n max: 10\n}\n```\n\n2. **Health checks**: Verify database before starting app\n```javascript\nconst healthCheck = async () => {\n try {\n await db.raw('SELECT 1');\n console.log('Database connected');\n } catch (err) {\n console.error('Database connection failed:', err);\n process.exit(1);\n }\n};\n```\n\n3. **Retry logic**: Auto-reconnect on temporary failures\n```javascript\nretry: {\n max: 3,\n backoff: 1000\n}\n```\n\n4. **Environment validation**: Check all required env vars exist\n```javascript\nconst required = ['DB_HOST', 'DB_PORT', 'DB_NAME', 'DB_USER', 'DB_PASSWORD'];\nfor (const key of required) {\n if (!process.env[key]) {\n throw new Error(`Missing required env var: ${key}`);\n }\n}\n```\n\n## Related Questions\n- How do I configure database connection pooling?\n- Database backup and restore procedures\n- Database performance optimization\n\n## Last Updated\nOctober 14, 2025" +}] +``` + +**FAQ Entry 3: General Troubleshooting** + +``` +Notion:notion-create-pages + +parent: { data_source_id: "collection://faq-db-uuid" } + +pages: [{ + properties: { + "Question": "What's the first thing I should check when deployment fails?", + "Category": "Troubleshooting", + "Tags": "deployment, debugging, best-practices", + "date:Last Reviewed:start": "2025-10-14", + "date:Last Reviewed:is_datetime": 0 + }, + content: "## Short Answer\n\n**Always check the logs first.** Logs contain error messages that point you directly to the problem. Use `pm2 logs`, `docker logs`, or check your application's log files.\n\n## Detailed Explanation\n\nLogs are your first and most important debugging tool. They show:\n- Exact error messages\n- Stack traces\n- Timing information\n- Configuration issues\n- Dependency problems\n\nMost deployment issues can be diagnosed and fixed by reading the logs carefully.\n\n## How to Check Logs\n\n### PM2\n\n```bash\n# View all logs\npm2 logs\n\n# View logs for specific app\npm2 logs app-name\n\n# View only errors\npm2 logs --err\n\n# Follow logs in real-time\npm2 logs --lines 100\n```\n\n### Docker\n\n```bash\n# View logs\ndocker logs container-name\n\n# Follow logs\ndocker logs -f container-name\n\n# Last 100 lines\ndocker logs --tail 100 container-name\n\n# With timestamps\ndocker logs -t container-name\n```\n\n### Application Logs\n\n```bash\n# Tail application logs\ntail -f /var/log/app/app.log\ntail -f /var/log/app/error.log\n\n# Search logs for errors\ngrep -i error /var/log/app/*.log\n\n# View logs with context\ngrep -B 5 -A 5 \"ERROR\" app.log\n```\n\n## Systematic Troubleshooting Approach\n\n### 1. Check the logs\n- Read error messages carefully\n- Note the exact error type and message\n- Check timestamps to find when error occurred\n\n### 2. Verify configuration\n- Environment variables set correctly?\n- Configuration files present and valid?\n- Paths and file permissions correct?\n\n### 3. Check dependencies\n- All packages installed? (`node_modules` present?)\n- Correct versions installed?\n- Any native module compilation errors?\n\n### 4. Verify environment\n- Required services running (database, Redis, etc.)?\n- Ports available?\n- Network connectivity working?\n\n### 5. Test components individually\n- Can you connect to database manually?\n- Can you run application locally?\n- Do health check endpoints work?\n\n### 6. Check recent changes\n- What changed since last successful deployment?\n- New dependencies added?\n- Configuration modified?\n- Environment differences?\n\n## Common Error Patterns\n\n### \"Module not found\"\n```bash\n# Solution: Install dependencies\nnpm install\n# or\nnpm ci\n```\n\n### \"Permission denied\"\n```bash\n# Solution: Fix file permissions\nchmod +x start.sh\nsudo chown -R appuser:appuser /app\n```\n\n### \"Address already in use\"\n```bash\n# Solution: Kill process on port\nlsof -ti:3000 | xargs kill -9\n```\n\n### \"Cannot connect to...\"\n```bash\n# Solution: Verify service is running and reachable\ntelnet service-host port\n```\n\n## Debugging Tools\n\n### Log Aggregation\n- **PM2**: Built-in log management\n- **Docker**: Centralized logging with log drivers\n- **ELK Stack**: Elasticsearch, Logstash, Kibana for large scale\n- **CloudWatch**: For AWS deployments\n\n### Monitoring\n- **PM2 Monit**: `pm2 monit` for real-time metrics\n- **Docker Stats**: `docker stats` for resource usage\n- **System metrics**: `top`, `htop` for CPU/memory\n\n### Network Debugging\n```bash\n# Check open connections\nnetstat -tulpn\n\n# Check specific port\nlsof -i :3000\n\n# Test connectivity\ncurl http://localhost:3000/health\n```\n\n## When to Escalate\n\nEscalate to senior team member if:\n- Error message is cryptic and not Google-able\n- Problem persists after trying obvious solutions\n- Production is affected\n- Data integrity may be compromised\n- Security-related errors\n\n## Prevention\n\n1. **Automated health checks** before deployment\n2. **Staged deployments** (dev → staging → production)\n3. **Rollback plan** ready for quick recovery\n4. **Monitoring and alerts** for early problem detection\n5. **Deployment checklist** to ensure all steps followed\n\n## Related Questions\n- How do I set up log aggregation?\n- Deployment checklist and best practices\n- How to roll back a failed deployment?\n\n## Last Updated\nOctober 14, 2025" +}] +``` + +### Step 5: Update FAQ Index + +``` +Notion:fetch +id: "faq-index-page-id" +``` + +``` +Notion:notion-update-page +page_id: "faq-index-page-id" +command: "insert_content_after" +selection_with_ellipsis: "## Deployment & Troubleshooting..." +new_str: " +- Why does deployment fail with 'port already in use' error? +- Why do I get 'cannot connect to database' errors? +- What's the first thing I should check when deployment fails? +" +``` + +## Summary Provided to User + +```markdown +## FAQ Entries Created ✅ + +### Conversation Captured +Transformed deployment troubleshooting conversation into 3 comprehensive FAQ entries. + +### FAQ Entries Created + +1. **Port Already in Use Error** + - Problem: Deployment fails due to occupied port + - Solutions: Kill process, use PM2, check node processes + - Prevention: Use process managers, graceful shutdown + - Link: Port already in use FAQ + +2. **Database Connection Errors** + - Problem: Cannot connect to database during deployment + - Troubleshooting: 4-step systematic approach + - Solutions: By error type (credentials, network, limits) + - Prevention: Connection pooling, health checks, retry logic + - Link: Database connection FAQ + +3. **General Troubleshooting Approach** + - Best practice: Always check logs first + - Tools: PM2 logs, Docker logs, application logs + - Systematic approach: 6-step debugging process + - Common errors: Module not found, permissions, ports + - Link: Troubleshooting first steps FAQ + +### Content Structure + +Each FAQ includes: +✅ Short answer (quick reference) +✅ Detailed explanation (full context) +✅ Multiple solution options +✅ Code examples and commands +✅ Prevention strategies +✅ Links to related FAQs +✅ Last updated date + +### Database Integration + +- ✅ Added to FAQ database (not standalone pages) +- ✅ Properties set: Category (Troubleshooting), Tags, Date +- ✅ FAQ index page updated with links +- ✅ Cross-references between related questions + +### Value Added + +**From conversation** → **To structured knowledge**: +- Casual Q&A → Professional FAQ entries +- Implicit knowledge → Explicit documentation +- One-time help → Reusable resource +- 5-minute conversation → Permanent team knowledge + +**Future benefit**: +- Team can self-serve on common issues +- New team members can onboard faster +- Reduces repeated questions +- Improves troubleshooting speed + +**Knowledge captured and preserved! 📚** +``` + +## Key Features Demonstrated + +### Content Extraction +- Identified distinct Q&A pairs from conversation +- Extracted commands, solutions, and best practices +- Preserved troubleshooting workflows + +### FAQ Structure +- Short answer for quick reference +- Detailed explanation for understanding +- Multiple solutions (Options 1, 2, 3) +- Code examples with comments +- Prevention strategies +- Related questions linking + +### Database Integration +- Created in FAQ database (not standalone) +- Set appropriate properties (category, tags, date) +- Updated index page with links +- Cross-referenced related questions + +### Value Transformation +- Converted informal conversation to professional docs +- Added structure, examples, and prevention tips +- Made knowledge searchable and discoverable +- Created permanent team resource + +Perfect for: FAQ creation, knowledge preservation, team documentation + diff --git a/plugins/notion/skills/notion-knowledge-capture/examples/decision-capture.md b/plugins/notion/skills/notion-knowledge-capture/examples/decision-capture.md new file mode 100644 index 0000000..0bc07cf --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/examples/decision-capture.md @@ -0,0 +1,126 @@ +# Example: Decision Record Capture + +**User Request**: "Document our decision to move from REST to GraphQL API" + +## Workflow + +### 1. Extract Decision from Context +From conversation, identified: +- **Decision**: Migrate customer-facing API from REST to GraphQL +- **Context**: REST endpoints becoming unwieldy (50+ endpoints) +- **Alternatives**: Keep REST, try gRPC, adopt GraphQL +- **Rationale**: Better client experience, type safety, single endpoint + +### 2. Find Decision Log +``` +Notion:search +query: "architecture decisions" or "ADR" +``` + +Found: "Architecture Decision Records" database + +### 3. Fetch Schema +``` +Notion:fetch +``` + +Got properties: Decision (title), Date, Status, Domain, Impact, Deciders, Stakeholders + +### 4. Create Decision Record +``` +Notion:notion-create-pages +parent: { data_source_id: "decision-log-collection-id" } +pages: [{ + properties: { + "Decision": "Migrate to GraphQL API", + "date:Date:start": "2025-10-16", + "date:Date:is_datetime": 0, + "Status": "Accepted", + "Domain": "Architecture", + "Impact": "High" + }, + content: "[Full decision record with context, rationale, alternatives...]" +}] +``` + +**Content sample**: + +```markdown +# Migrate Customer-Facing API to GraphQL + +## Context +Our REST API has grown to 50+ endpoints with inconsistent patterns. Frontend and mobile teams request new endpoints frequently, leading to: +- API bloat and maintenance burden +- Over-fetching/under-fetching data +- Slow iteration on client features +- Inconsistent error handling + +## Decision +Migrate customer-facing API from REST to GraphQL while maintaining REST for internal services. + +## Rationale +**Why GraphQL**: +- Clients fetch exactly what they need (no over/under-fetching) +- Single endpoint, self-documenting schema +- Type safety with code generation +- Better developer experience +- Industry standard for client-facing APIs + +## Options Considered + +### Option A: Keep REST (Status Quo) +**Pros**: No migration cost, team familiar +**Cons**: Doesn't solve core problems, maintenance keeps growing +**Decision**: Rejected - doesn't address root cause + +### Option B: gRPC +**Pros**: Performance, strong typing +**Cons**: Browser support issues, steeper learning curve, not designed for client-facing +**Decision**: Rejected - better for internal services + +### Option C: GraphQL +**Pros**: Solves over/under-fetching, great DX, strong ecosystem +**Cons**: Learning curve, caching complexity, migration effort +**Decision**: **Accepted** + +## Consequences + +**Positive**: +- Faster frontend/mobile development +- Reduced API maintenance burden +- Better type safety and tooling +- Single endpoint simplifies deployment + +**Negative**: +- 3-4 month migration timeline +- Team needs training +- Need to solve caching strategy +- Monitoring/debugging patterns different + +## Implementation Plan +1. GraphQL server setup (Apollo Server) +2. Schema design workshop +3. Gradual migration (start with new features) +4. Dual-run period (both REST and GraphQL) +5. Deprecate old REST endpoints + +**Timeline**: Start Q4 2025, complete Q1 2026 +**Owner**: Backend team lead + +## Success Metrics +- API response times improve 30% +- Client fetch efficiency (less data transferred) +- Reduced new endpoint requests +- Developer satisfaction scores +``` + +### 5. Make Discoverable +Added link from Architecture Wiki and notified team in Slack. + +## Key Success Factors +- Captured decision while context fresh +- Documented alternatives considered +- Included both pros and cons +- Clear implementation plan +- Saved to decision log for future reference +- Made discoverable for team diff --git a/plugins/notion/skills/notion-knowledge-capture/examples/how-to-guide.md b/plugins/notion/skills/notion-knowledge-capture/examples/how-to-guide.md new file mode 100644 index 0000000..6b12e4d --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/examples/how-to-guide.md @@ -0,0 +1,118 @@ +# Example: How-To Guide from Discussion + +**User Request**: "Save our discussion about deploying to production as a how-to guide" + +## Workflow + +### 1. Extract Content from Chat +From conversation, identified: +- Deployment prerequisites +- Step-by-step procedure +- Common issues and solutions +- Best practices and tips + +### 2. Structure as How-To +Organized into: +- Overview & prerequisites +- Numbered deployment steps +- Verification steps +- Troubleshooting section +- Related resources + +### 3. Find Location +``` +Notion:search +query: "deployment documentation" +``` +Found: Engineering Wiki → Deployment section + +### 4. Create Guide +``` +Notion:notion-create-pages +parent: { page_id: "deployment-section-id" } +``` + +## Output + +```markdown +# How to Deploy to Production + +## Overview +Production deployment using GitHub Actions with zero-downtime rolling updates. +**Time Required**: 15-20 minutes | **Complexity**: Intermediate + +## Prerequisites +- [ ] PR approved and merged to main +- [ ] All CI tests passing +- [ ] Database migrations reviewed +- [ ] Feature flags configured + +## Deployment Steps + +1. **Verify main branch is ready** + ```bash + git checkout main && git pull + ``` + +2. **Tag release** + ```bash + git tag -a v1.2.3 -m "Release v1.2.3" + git push origin v1.2.3 + ``` + +3. **Trigger deployment** + - GitHub Actions auto-starts from tag push + - Monitor: https://github.com/org/repo/actions + +4. **Database migrations** (if needed) + - Auto-run in GitHub Actions + - Check logs for completion + +5. **Verify deployment** + - Wait for health checks (2-3 min) + - Test key endpoints + - Check error rates in Datadog + +## Verification Checklist +- [ ] All pods healthy in k8s dashboard +- [ ] Error rate < 0.1% in last 10 min +- [ ] Response time p95 < 500ms +- [ ] Test login flow +- [ ] Check Slack #alerts channel + +## Troubleshooting + +**Health checks failing** +→ Check pod logs: `kubectl logs -f deployment/api -n production` + +**Migration errors** +→ Rollback: Revert tag, migrations auto-rollback + +**High error rate** +→ Emergency rollback: Previous tag auto-deploys via GitHub Actions + +## Best Practices +- Deploy during low-traffic hours (2-4am PST) +- Have 2 engineers available +- Monitor for 30 min post-deploy +- Update #engineering Slack with deploy notice + +## Related Docs +- Rollback Procedure +- Database Migration Guide +``` + +### 5. Make Discoverable +``` +Notion:notion-update-page +page_id: "engineering-wiki-homepage" +command: "insert_content_after" +``` +Added link in Engineering Wiki → How-To Guides section + +## Key Success Factors +- Captured tribal knowledge from discussion +- Structured as actionable steps +- Included troubleshooting from experience +- Made discoverable by linking from wiki index +- Added metadata (time, complexity) diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/database-best-practices.md b/plugins/notion/skills/notion-knowledge-capture/reference/database-best-practices.md new file mode 100644 index 0000000..b27ac3f --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/database-best-practices.md @@ -0,0 +1,112 @@ +# Database Best Practices + +General guidance for creating and maintaining knowledge capture databases. + +## Core Principles + +### 1. Keep It Simple +- Start with core properties +- Add more only when needed +- Don't over-engineer + +### 2. Use Consistent Naming +- Title property for main identifier +- Status for lifecycle tracking +- Tags for flexible categorization +- Owner for accountability + +### 3. Include Metadata +- Created/Updated timestamps +- Owner or maintainer +- Last reviewed dates +- Status indicators + +### 4. Enable Discovery +- Use tags liberally +- Create helpful views +- Link related content +- Use clear titles + +### 5. Plan for Scale +- Consider filters early +- Use relations for connections +- Think about search +- Organize with categories + +## Creating a Database + +### Using `Notion:notion-create-database` + +Example for documentation database: + +```javascript +{ + "parent": {"page_id": "wiki-page-id"}, + "title": [{"text": {"content": "Team Documentation"}}], + "properties": { + "Type": { + "select": { + "options": [ + {"name": "How-To", "color": "blue"}, + {"name": "Concept", "color": "green"}, + {"name": "Reference", "color": "gray"}, + {"name": "FAQ", "color": "yellow"} + ] + } + }, + "Category": { + "select": { + "options": [ + {"name": "Engineering", "color": "red"}, + {"name": "Product", "color": "purple"}, + {"name": "Design", "color": "pink"} + ] + } + }, + "Tags": {"multi_select": {"options": []}}, + "Owner": {"people": {}}, + "Status": { + "select": { + "options": [ + {"name": "Draft", "color": "gray"}, + {"name": "Final", "color": "green"}, + {"name": "Deprecated", "color": "red"} + ] + } + } + } +} +``` + +### Fetching Database Schema + +Before creating pages, always fetch database to get schema: + +``` +Notion:fetch +id: "database-url-or-id" +``` + +This returns the exact property names and types to use. + +## Database Selection Guide + +| Need | Use This Database | +|------|-------------------| +| General documentation | [Documentation Database](documentation-database.md) | +| Track decisions | [Decision Log](decision-log-database.md) | +| Q&A knowledge base | [FAQ Database](faq-database.md) | +| Team-specific content | [Team Wiki](team-wiki-database.md) | +| Step-by-step guides | [How-To Guide Database](how-to-guide-database.md) | +| Incident/project learnings | [Learning Database](learning-database.md) | + +## Tips + +1. **Start with general documentation database** - most flexible +2. **Add specialized databases** as needs emerge (FAQ, Decisions) +3. **Use relations** to connect related docs +4. **Create views** for common use cases +5. **Review properties** quarterly - remove unused ones +6. **Document the schema** in database description +7. **Train team** on property usage and conventions + diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/decision-log-database.md b/plugins/notion/skills/notion-knowledge-capture/reference/decision-log-database.md new file mode 100644 index 0000000..2a7e6e4 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/decision-log-database.md @@ -0,0 +1,58 @@ +# Decision Log Database (ADR - Architecture Decision Records) + +**Purpose**: Track important decisions with context and rationale. + +## Schema + +| Property | Type | Options | Purpose | +|----------|------|---------|---------| +| **Decision** | title | - | What was decided | +| **Date** | date | - | When decision was made | +| **Status** | select | Proposed, Accepted, Superseded, Deprecated | Current decision status | +| **Domain** | select | Architecture, Product, Business, Design, Operations | Decision category | +| **Impact** | select | High, Medium, Low | Expected impact level | +| **Deciders** | people | - | Who made the decision | +| **Stakeholders** | people | - | Who's affected by decision | +| **Related Decisions** | relation | Links to other decisions | Context and dependencies | + +## Usage + +``` +Create decision records with properties: +{ + "Decision": "Use PostgreSQL for Primary Database", + "Date": "2025-10-15", + "Status": "Accepted", + "Domain": "Architecture", + "Impact": "High", + "Deciders": [tech_lead, architect], + "Stakeholders": [eng_team] +} +``` + +## Content Template + +Each decision page should include: +- **Context**: Why this decision was needed +- **Decision**: What was decided +- **Rationale**: Why this option was chosen +- **Options Considered**: Alternatives and trade-offs +- **Consequences**: Expected outcomes (positive and negative) +- **Implementation**: How decision will be executed + +## Views + +**Recent Decisions**: Sort by Date descending +**Active Decisions**: Filter where Status = "Accepted" +**By Domain**: Group by Domain +**High Impact**: Filter where Impact = "High" +**Pending**: Filter where Status = "Proposed" + +## Best Practices + +1. **Document immediately**: Record decisions when made, while context is fresh +2. **Include alternatives**: Show what was considered and why it wasn't chosen +3. **Track superseded decisions**: Update status when decisions change +4. **Link related decisions**: Use relations to show dependencies +5. **Review periodically**: Check if old decisions are still valid + diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/documentation-database.md b/plugins/notion/skills/notion-knowledge-capture/reference/documentation-database.md new file mode 100644 index 0000000..ebb2833 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/documentation-database.md @@ -0,0 +1,93 @@ +# General Documentation Database + +**Purpose**: Store all types of documentation in a searchable, organized database. + +## Schema + +| Property | Type | Options | Purpose | +|----------|------|---------|---------| +| **Title** | title | - | Document name | +| **Type** | select | How-To, Concept, Reference, FAQ, Decision, Post-Mortem | Categorize content type | +| **Category** | select | Engineering, Product, Design, Operations, General | Organize by department/topic | +| **Tags** | multi_select | - | Additional categorization (languages, tools, topics) | +| **Status** | select | Draft, In Review, Final, Deprecated | Track document lifecycle | +| **Owner** | people | - | Document maintainer | +| **Created** | created_time | - | Auto-populated creation date | +| **Last Updated** | last_edited_time | - | Auto-populated last edit | +| **Last Reviewed** | date | - | Manual review tracking | + +## Usage + +``` +Create pages with properties: +{ + "Title": "How to Deploy to Production", + "Type": "How-To", + "Category": "Engineering", + "Tags": "deployment, production, DevOps", + "Status": "Final", + "Owner": [current_user], + "Last Reviewed": "2025-10-01" +} +``` + +## Views + +**By Type**: Group by Type property +**By Category**: Group by Category property +**Recent Updates**: Sort by Last Updated descending +**Needs Review**: Filter where Last Reviewed > 90 days ago +**Draft Docs**: Filter where Status = "Draft" + +## Creating This Database + +Use `Notion:notion-create-database`: + +```javascript +{ + "parent": {"page_id": "wiki-page-id"}, + "title": [{"text": {"content": "Team Documentation"}}], + "properties": { + "Type": { + "select": { + "options": [ + {"name": "How-To", "color": "blue"}, + {"name": "Concept", "color": "green"}, + {"name": "Reference", "color": "gray"}, + {"name": "FAQ", "color": "yellow"} + ] + } + }, + "Category": { + "select": { + "options": [ + {"name": "Engineering", "color": "red"}, + {"name": "Product", "color": "purple"}, + {"name": "Design", "color": "pink"} + ] + } + }, + "Tags": {"multi_select": {"options": []}}, + "Owner": {"people": {}}, + "Status": { + "select": { + "options": [ + {"name": "Draft", "color": "gray"}, + {"name": "Final", "color": "green"}, + {"name": "Deprecated", "color": "red"} + ] + } + } + } +} +``` + +## Best Practices + +1. **Start with this schema** - most flexible for general documentation +2. **Use relations** to connect related docs +3. **Create views** for common use cases +4. **Review properties** quarterly - remove unused ones +5. **Document the schema** in database description +6. **Train team** on property usage and conventions + diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/faq-database.md b/plugins/notion/skills/notion-knowledge-capture/reference/faq-database.md new file mode 100644 index 0000000..32457dd --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/faq-database.md @@ -0,0 +1,57 @@ +# FAQ Database + +**Purpose**: Organize frequently asked questions with answers. + +## Schema + +| Property | Type | Options | Purpose | +|----------|------|---------|---------| +| **Question** | title | - | The question being asked | +| **Category** | select | Product, Engineering, Support, HR, General | Question topic | +| **Tags** | multi_select | - | Specific topics (auth, billing, onboarding, etc.) | +| **Answer Type** | select | Quick Answer, Detailed Guide, Link to Docs | Response format | +| **Last Reviewed** | date | - | When answer was verified | +| **Helpful Count** | number | - | Track usefulness (optional) | +| **Audience** | select | Internal, External, All | Who should see this | +| **Related Questions** | relation | Links to related FAQs | Connect similar topics | + +## Usage + +``` +Create FAQ entries with properties: +{ + "Question": "How do I reset my password?", + "Category": "Support", + "Tags": "authentication, password, login", + "Answer Type": "Quick Answer", + "Last Reviewed": "2025-10-01", + "Audience": "External" +} +``` + +## Content Template + +Each FAQ page should include: +- **Short Answer**: 1-2 sentence quick response +- **Detailed Explanation**: Full answer with context +- **Steps** (if applicable): Numbered procedure +- **Screenshots** (if helpful): Visual guidance +- **Related Questions**: Links to similar FAQs +- **Additional Resources**: External docs or videos + +## Views + +**By Category**: Group by Category +**Recently Updated**: Sort by Last Reviewed descending +**Needs Review**: Filter where Last Reviewed > 180 days ago +**External FAQs**: Filter where Audience contains "External" +**Popular**: Sort by Helpful Count descending (if tracking) + +## Best Practices + +1. **Use clear questions**: Write questions as users would ask them +2. **Provide quick answers**: Lead with the direct answer, then elaborate +3. **Link related FAQs**: Help users discover related information +4. **Review regularly**: Keep answers current and accurate +5. **Track what's helpful**: Use feedback to improve frequently accessed FAQs + diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/how-to-guide-database.md b/plugins/notion/skills/notion-knowledge-capture/reference/how-to-guide-database.md new file mode 100644 index 0000000..57593b1 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/how-to-guide-database.md @@ -0,0 +1,38 @@ +# How-To Guide Database + +**Purpose**: Procedural documentation for common tasks. + +## Schema + +| Property | Type | Options | Purpose | +|----------|------|---------|---------| +| **Title** | title | - | "How to [Task]" | +| **Complexity** | select | Beginner, Intermediate, Advanced | Skill level required | +| **Time Required** | number | - | Estimated minutes to complete | +| **Prerequisites** | relation | Links to other guides | Required knowledge | +| **Category** | select | Development, Deployment, Testing, Tools | Task category | +| **Last Tested** | date | - | When procedure was verified | +| **Tags** | multi_select | - | Technology/tool tags | + +## Usage + +``` +Create how-to guides with properties: +{ + "Title": "How to Set Up Local Development Environment", + "Complexity": "Beginner", + "Time Required": 30, + "Category": "Development", + "Last Tested": "2025-10-01", + "Tags": "setup, environment, docker" +} +``` + +## Best Practices + +1. **Use consistent naming**: Always start with "How to..." +2. **Test procedures**: Verify steps work before publishing +3. **Include time estimates**: Help users plan their time +4. **Link prerequisites**: Make dependencies clear +5. **Update regularly**: Re-test procedures when tools/systems change + diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/learning-database.md b/plugins/notion/skills/notion-knowledge-capture/reference/learning-database.md new file mode 100644 index 0000000..02f5f35 --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/learning-database.md @@ -0,0 +1,35 @@ +# Learning/Post-Mortem Database + +**Purpose**: Capture learnings from incidents, projects, or experiences. + +## Schema + +| Property | Type | Options | Purpose | +|----------|------|---------|---------| +| **Title** | title | - | Event or project name | +| **Date** | date | - | When it happened | +| **Type** | select | Incident, Project, Experiment, Retrospective | Learning type | +| **Severity** | select | Critical, Major, Minor | Impact level (for incidents) | +| **Team** | people | - | Who was involved | +| **Key Learnings** | number | - | Count of learnings | +| **Action Items** | relation | Links to tasks | Follow-up actions | + +## Content Template + +Each learning page should include: +- **What Happened**: Situation description +- **What Went Well**: Success factors +- **What Didn't Go Well**: Problems encountered +- **Root Causes**: Why things happened +- **Learnings**: Key takeaways +- **Action Items**: Improvements to implement + +## Best Practices + +1. **Blameless approach**: Focus on systems and processes, not individuals +2. **Document quickly**: Capture while memory is fresh +3. **Identify root causes**: Go beyond surface-level problems +4. **Create action items**: Turn learnings into improvements +5. **Follow up**: Track that action items are completed +6. **Share widely**: Make learnings accessible to entire team + diff --git a/plugins/notion/skills/notion-knowledge-capture/reference/team-wiki-database.md b/plugins/notion/skills/notion-knowledge-capture/reference/team-wiki-database.md new file mode 100644 index 0000000..e7b741f --- /dev/null +++ b/plugins/notion/skills/notion-knowledge-capture/reference/team-wiki-database.md @@ -0,0 +1,27 @@ +# Team Wiki Database + +**Purpose**: Centralized team knowledge and resources. + +## Schema + +| Property | Type | Options | Purpose | +|----------|------|---------|---------| +| **Title** | title | - | Page name | +| **Section** | select | Getting Started, Processes, Tools, Reference, Onboarding | Wiki organization | +| **Tags** | multi_select | - | Topic tags | +| **Owner** | people | - | Page maintainer | +| **Last Updated** | last_edited_time | - | Auto-tracked | +| **Visibility** | select | Public, Team Only, Confidential | Access level | + +## Usage + +Use for team-specific documentation that doesn't fit other databases. + +## Best Practices + +1. **Organize by sections**: Use clear top-level organization +2. **Assign owners**: Every page should have a maintainer +3. **Control visibility**: Set appropriate access levels +4. **Link extensively**: Connect related pages +5. **Keep current**: Regular reviews to remove outdated content + diff --git a/plugins/notion/skills/notion-meeting-intelligence/LICENSE.txt b/plugins/notion/skills/notion-meeting-intelligence/LICENSE.txt new file mode 100644 index 0000000..0871708 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/LICENSE.txt @@ -0,0 +1,7 @@ +Copyright 2025 Notion Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/notion/skills/notion-meeting-intelligence/SKILL.md b/plugins/notion/skills/notion-meeting-intelligence/SKILL.md new file mode 100644 index 0000000..f9d1dac --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/SKILL.md @@ -0,0 +1,60 @@ +--- +name: notion-meeting-intelligence +description: Prepare meeting materials with Notion context and Codex research; use when gathering context, drafting agendas/pre-reads, and tailoring materials to attendees. +metadata: + short-description: Prep meetings with Notion context and tailored agendas +--- + +# Meeting Intelligence + +Prep meetings by pulling Notion context, tailoring agendas/pre-reads, and enriching with Codex research. + +## Quick start +1) Confirm meeting goal, attendees, date/time, and decisions needed. +2) Gather context: search with `Notion:search`, then fetch with `Notion:fetch` (prior notes, specs, OKRs, decisions). +3) Pick the right template via `reference/template-selection-guide.md` (status, decision, planning, retro, 1:1, brainstorming). +4) Draft agenda/pre-read in Notion with `Notion:notion-create-pages`, embedding source links and owner/timeboxes. +5) Enrich with Codex research (industry insights, benchmarks, risks) and update the page with `Notion:notion-update-page` as plans change. + +## Workflow +### 0) If any MCP call fails because Notion MCP is not connected, pause and set it up: +1. Add the Notion MCP: + - `codex mcp add notion --url https://mcp.notion.com/mcp` +2. Enable remote MCP client: + - Set `[features].rmcp_client = true` in `config.toml` **or** run `codex --enable rmcp_client` +3. Log in with OAuth: + - `codex mcp login notion` + +After successful login, the user will have to restart codex. You should finish your answer and tell them so when they try again they can continue with Step 1. + +### 1) Gather inputs +- Ask for objective, desired outcomes/decisions, attendees, duration, date/time, and prior materials. +- Search Notion for relevant docs, past notes, specs, and action items (`Notion:search`), then fetch key pages (`Notion:fetch`). +- Capture blockers/risks and open questions up front. + +### 2) Choose format +- Status/update → status template. +- Decision/approval → decision template. +- Planning (sprint/project) → planning template. +- Retro/feedback → retrospective template. +- 1:1 → one-on-one template. +- Ideation → brainstorming template. +- Use `reference/template-selection-guide.md` to confirm. + +### 3) Build the agenda/pre-read +- Start from the chosen template in `reference/` and adapt sections (context, goals, agenda, owner/time per item, decisions, risks, prep asks). +- Include links to pulled Notion pages and any required pre-reading. +- Assign owners for each agenda item; call out timeboxes and expected outputs. + +### 4) Enrich with research +- Add concise Codex research where helpful: market/industry facts, benchmarks, risks, best practices. +- Keep claims cited with source links; separate fact from opinion. + +### 5) Finalize and share +- Add next steps and owners for follow-ups. +- If tasks arise, create/link tasks in the relevant Notion database. +- Update the page via `Notion:notion-update-page` when details change; keep a brief changelog if multiple edits. + +## References and examples +- `reference/` — template picker and meeting templates (e.g., `template-selection-guide.md`, `status-update-template.md`, `decision-meeting-template.md`, `sprint-planning-template.md`, `one-on-one-template.md`, `retrospective-template.md`, `brainstorming-template.md`). +- `examples/` — end-to-end meeting preps (e.g., `executive-review.md`, `project-decision.md`, `sprint-planning.md`, `customer-meeting.md`). diff --git a/plugins/notion/skills/notion-meeting-intelligence/agents/openai.yaml b/plugins/notion/skills/notion-meeting-intelligence/agents/openai.yaml new file mode 100644 index 0000000..f4dd6a8 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/agents/openai.yaml @@ -0,0 +1,14 @@ +interface: + display_name: "Notion Meeting Intelligence" + short_description: "Prep meetings with Notion context and tailored agendas" + icon_small: "./assets/notion-small.svg" + icon_large: "./assets/notion.png" + default_prompt: "Prepare this meeting from Notion context with a brief, agenda, decisions needed, and open questions." + +dependencies: + tools: + - type: "mcp" + value: "notion" + description: "Notion MCP server" + transport: "streamable_http" + url: "https://mcp.notion.com/mcp" diff --git a/plugins/notion/skills/notion-meeting-intelligence/assets/notion-small.svg b/plugins/notion/skills/notion-meeting-intelligence/assets/notion-small.svg new file mode 100644 index 0000000..455d4be --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/assets/notion-small.svg @@ -0,0 +1,11 @@ + diff --git a/plugins/notion/skills/notion-meeting-intelligence/assets/notion.png b/plugins/notion/skills/notion-meeting-intelligence/assets/notion.png new file mode 100644 index 0000000..085023c Binary files /dev/null and b/plugins/notion/skills/notion-meeting-intelligence/assets/notion.png differ diff --git a/plugins/notion/skills/notion-meeting-intelligence/evaluations/README.md b/plugins/notion/skills/notion-meeting-intelligence/evaluations/README.md new file mode 100644 index 0000000..eb77e17 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/evaluations/README.md @@ -0,0 +1,101 @@ +# Meeting Intelligence Skill Evaluations + +Evaluation scenarios for testing the Meeting Intelligence skill across different Codex models. + +## Purpose + +These evaluations ensure the Meeting Intelligence skill: +- Gathers context from Notion workspace +- Enriches with Codex research appropriately +- Creates both internal pre-reads and external agendas +- Distinguishes between Notion facts and Codex insights +- Works consistently across Haiku, Sonnet, and Opus + +## Evaluation Files + +### decision-meeting-prep.json +Tests preparation for a decision-making meeting. + +**Scenario**: Prep for database migration decision meeting +**Key Behaviors**: +- Searches Notion for migration context (specs, discussions, options) +- Fetches 2-3 relevant pages +- Enriches with Codex research (decision frameworks, migration best practices) +- Creates comprehensive internal pre-read with recommendation +- Creates clean, professional external agenda +- Clearly distinguishes Notion facts from Codex insights +- Cross-links both documents + +### status-meeting-prep.json +Tests preparation for a status update or review meeting. + +**Scenario**: Prep for project status review +**Key Behaviors**: +- Gathers project metrics and progress from Notion +- Fetches relevant pages (roadmap, tasks, milestones) +- Adds Codex context (industry benchmarks, best practices) +- Creates internal pre-read with honest assessment +- Creates external agenda with structured flow +- Includes source citations using mention-page tags +- Time-boxes agenda items + +## Running Evaluations + +1. Enable the `meeting-intelligence` skill +2. Submit the query from the evaluation file +3. Verify the skill searches Notion first (not Codex research) +4. Check that TWO documents are created (internal + external) +5. Verify Codex enrichment adds value without replacing Notion content +6. Test with Haiku, Sonnet, and Opus + +## Expected Skill Behaviors + +Meeting Intelligence evaluations should verify: + +### Notion Context Gathering +- Searches workspace for relevant context first +- Fetches specific pages (not generic) +- Extracts key information from Notion content +- Cites sources using mention-page tags + +### Codex Research Integration +- Adds industry context, frameworks, or best practices +- Enrichment is relevant and valuable (not filler) +- Clearly distinguishes Notion facts from Codex insights +- Research complements (doesn't replace) Notion content + +### Two-Document Creation +- **Internal Pre-Read**: Comprehensive, includes strategy, recommendations, detailed pros/cons +- **External Agenda**: Professional, focused on meeting flow, no internal strategy +- Both documents are clearly labeled +- Documents are cross-linked + +### Document Quality +- Pre-read follows structure: Overview → Background → Current Status → Context & Insights → Discussion Points +- Agenda follows structure: Details → Objective → Agenda Items (with times) → Decisions → Actions → Resources +- Titles include date or meeting context +- Content is actionable and meeting-ready + +## Creating New Evaluations + +When adding Meeting Intelligence evaluations: + +1. **Test different meeting types** - Decision, status, brainstorm, 1:1, sprint planning, retrospective +2. **Vary complexity** - Simple updates vs. complex strategic decisions +3. **Test with/without Notion content** - Rich workspace vs. minimal existing pages +4. **Verify enrichment value** - Is Codex research genuinely helpful? +5. **Check internal/external distinction** - Is sensitive info kept in pre-read only? + +## Example Success Criteria + +**Good** (specific, testable): +- "Creates TWO documents (internal pre-read + external agenda)" +- "Internal pre-read marked 'INTERNAL ONLY' or 'For team only'" +- "Cites at least 2-3 Notion pages using mention-page tags" +- "Agenda includes time allocations for each section" +- "Codex enrichment includes decision frameworks or best practices" + +**Bad** (vague, untestable): +- "Creates meeting materials" +- "Gathers context effectively" +- "Prepares well" diff --git a/plugins/notion/skills/notion-meeting-intelligence/evaluations/decision-meeting-prep.json b/plugins/notion/skills/notion-meeting-intelligence/evaluations/decision-meeting-prep.json new file mode 100644 index 0000000..a440771 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/evaluations/decision-meeting-prep.json @@ -0,0 +1,35 @@ +{ + "name": "Decision Meeting Preparation", + "skills": ["meeting-intelligence"], + "query": "Prep for tomorrow's meeting where we need to decide on our database migration approach. Create both an internal pre-read for the team and an agenda for the meeting.", + "expected_behavior": [ + "Step 1: Uses Notion:search to find context about database migration (project pages, technical specs, previous discussions, options analysis)", + "Step 2: Fetches at least 2-3 relevant pages using Notion:fetch to gather information from Notion", + "Step 3: Identifies the decision to be made and available options from fetched Notion content", + "Step 4: Enriches with Codex research - adds decision-making frameworks (e.g., cost-benefit analysis, risk assessment), technical context for migration approaches, best practices for database migrations", + "Step 5: Distinguishes Notion facts from Codex insights in synthesis", + "Step 6: Creates INTERNAL PRE-READ using Notion:notion-create-pages with title like 'INTERNAL: Database Migration Decision - Pre-Read - [Date]'", + "Step 6a: Internal pre-read includes: Meeting overview, background context (from Notion), current status and technical details, context & insights (from Codex research on migration best practices), decision options with detailed pros/cons, recommendation with rationale, what we need from meeting", + "Step 6b: Internal pre-read marked clearly as 'INTERNAL ONLY' or 'For team only'", + "Step 7: Creates EXTERNAL AGENDA using Notion:notion-create-pages with title like 'Meeting Agenda: Database Migration Decision - [Date]'", + "Step 7a: External agenda includes: Meeting details, objective (clear decision to make), agenda items with time allocations, discussion topics, decisions needed, action items section (empty), related resources with link to pre-read", + "Step 7b: External agenda is clean, professional, focused (no internal strategy or detailed pros/cons)", + "Step 8: Links both documents together (agenda mentions pre-read, pre-read mentions agenda)", + "Both documents link to source pages using " + ], + "success_criteria": [ + "TWO documents are created (internal pre-read + external agenda), not just one", + "Internal pre-read is comprehensive with: Notion context + Codex insights + detailed pros/cons + recommendation", + "Internal pre-read is clearly marked 'INTERNAL' or 'For team only'", + "External agenda is professional, structured, focused on meeting flow (not internal strategy)", + "Codex enrichment is present and adds value (decision frameworks, migration best practices, risk patterns)", + "Notion facts are clearly sourced, Codex insights are distinguished", + "At least 2-3 Notion source pages are cited using mention-page tags", + "Internal pre-read follows structure from SKILL.md Step 5 (Meeting Overview → Background → Current Status → Context & Insights → Key Discussion Points → What We Need)", + "External agenda follows structure from SKILL.md Step 6 (Meeting Details → Objective → Agenda Items → Discussion Topics → Decisions Needed → Action Items → Related Resources)", + "Documents are cross-linked (pre-read mentions agenda, agenda mentions pre-read)", + "Meeting date is included in both titles", + "Uses correct tool names (Notion:search, Notion:fetch, Notion:notion-create-pages for BOTH documents)" + ] +} + diff --git a/plugins/notion/skills/notion-meeting-intelligence/evaluations/status-meeting-prep.json b/plugins/notion/skills/notion-meeting-intelligence/evaluations/status-meeting-prep.json new file mode 100644 index 0000000..99302de --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/evaluations/status-meeting-prep.json @@ -0,0 +1,35 @@ +{ + "name": "Status Update Meeting Preparation", + "skills": ["meeting-intelligence", "task-manager"], + "query": "Prep for Friday's project status meeting on the Mobile App Redesign project. Create both an internal pre-read and an external agenda.", + "expected_behavior": [ + "Step 1: Uses Notion:search to find Mobile App Redesign project page", + "Step 2: Fetches project page using Notion:fetch to get current status and context", + "Step 3: Uses Notion:search to find tasks database", + "Step 4: Queries task database using Notion:notion-query-data-sources for project tasks (WHERE Project = 'Mobile App Redesign')", + "Step 5: Analyzes task data: calculates completion %, identifies completed work, in-progress items, and blockers", + "Step 6: Enriches with Codex research - adds project management insights (velocity trends, risk patterns, common project pitfalls), suggests discussion frameworks if risks identified, provides context on timeline implications", + "Step 7: Creates INTERNAL PRE-READ using Notion:notion-create-pages with title 'INTERNAL: Mobile App Redesign Status - Pre-Read - [Date]'", + "Step 7a: Internal pre-read includes: Project overview, current status with metrics (from Notion/tasks), progress summary with specifics, context & insights (Codex research on project health patterns), honest assessment of challenges/risks, what we need from meeting", + "Step 7b: Internal pre-read contains detailed metrics, blockers, and strategic considerations", + "Step 8: Creates EXTERNAL AGENDA using Notion:notion-create-pages with title 'Meeting Agenda: Mobile App Redesign Status Update - [Date]'", + "Step 8a: External agenda uses Status Update structure: Meeting Details → Objective → Agenda Items (timed) → Discussion Topics → Action Items", + "Step 8b: External agenda is concise, professional, focuses on meeting flow (summary-level metrics only)", + "Step 9: Links both documents together", + "Both documents link to project page and task database using and " + ], + "success_criteria": [ + "TWO documents are created (internal pre-read + external agenda)", + "Internal pre-read contains: Detailed metrics from task query, honest assessment of blockers/risks, Codex insights on project health patterns, strategic considerations", + "Internal pre-read is marked 'INTERNAL' or 'For team only'", + "External agenda is professional status update format with: Overall status, progress summary (high-level), upcoming work, discussion topics, action items section", + "External agenda does NOT contain detailed risk analysis or internal strategic discussion (that's in pre-read)", + "Progress metrics are accurate based on Notion:notion-query-data-sources results (e.g., X complete, Y in progress, Z blocked)", + "Blockers identified from task query are included in internal pre-read with full context", + "Codex enrichment adds value (project management insights, risk patterns, velocity analysis)", + "Links to project page and task database using mention tags", + "Documents are cross-linked for easy navigation", + "Uses correct tool names (Notion:search, Notion:fetch, Notion:notion-query-data-sources, Notion:notion-create-pages for BOTH documents)" + ] +} + diff --git a/plugins/notion/skills/notion-meeting-intelligence/examples/customer-meeting.md b/plugins/notion/skills/notion-meeting-intelligence/examples/customer-meeting.md new file mode 100644 index 0000000..ba65ff3 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/examples/customer-meeting.md @@ -0,0 +1,125 @@ +# Example: Customer Meeting Prep + +**User Request**: "Prep for sales meeting with Acme Corp on Thursday" + +## Workflow + +### 1. Search Customer Info +``` +Notion:search +query: "Acme Corp" +``` + +Found: +- CRM notes from initial call +- Product demo feedback +- Pricing proposal +- Competitor analysis + +### 2. Fetch Details +``` +Notion:fetch (4 pages) +``` + +**Key context**: +- **Company**: 500-person fintech startup, Series B +- **Pain Points**: Manual workflows, data silos, scaling issues +- **Decision Makers**: CTO (technical), VP Product (user needs), CFO (budget) +- **Timeline**: Need solution by Q1 +- **Budget**: $50-75K/year range +- **Competition**: Evaluating us vs. Competitor A and Competitor B + +### 3. Add Research Context +Codex research added: +- Fintech industry compliance requirements +- Common integration needs for financial products +- Typical procurement processes for Series B companies + +### 4. Create Internal Pre-Read +``` +Notion:notion-create-pages +pages: [{ + properties: { + title: "Acme Corp Meeting - Pre-Read (Internal)" + }, + content: "# Acme Corp Meeting - Pre-Read (Internal) + +## Company Overview +**Company**: Acme Corp (500-person fintech, Series B) +**Decision Makers**: CTO, VP Product, CFO +Source: CRM Notes + +## Their Pain Points → Our Solutions +- Manual workflows → Workflow automation +- Data silos → API integrations +- Scaling issues → Enterprise features + +## Competitive Position +**Our advantages over Competitor A/B**: +- Better integration ecosystem (Stripe, Plaid) +- Stronger compliance features +- Faster implementation (4 weeks vs 12 weeks) + +## Pricing Strategy +Recommend: $60K/year (mid-range of their budget) + +## Potential Objections & Responses +..." +}] +``` + +### 5. Create Meeting Agenda +``` +Notion:notion-create-pages +pages: [{ + properties: { + title: "Acme Corp - Product Demo & Q&A" + }, + content: "# Acme Corp - Product Demo & Q&A + +## Meeting Details +**Date**: Thursday, Oct 17, 2025 @ 2pm PST +**Attendees**: CTO, VP Product, CFO (Acme) | Sales Lead, Solutions Engineer (Us) +**Duration**: 60 minutes + +## Objective +Demonstrate how our platform solves Acme's workflow automation and integration challenges. + +## Agenda + +**1. Introductions** (5 min) + +**2. Understand Current Workflow** (10 min) +- Current pain points +- Integration requirements +- Success criteria + +**3. Product Demo** (25 min) +- Workflow automation capabilities +- API integrations (Stripe, Plaid) +- Security & compliance features + +**4. Pricing & Implementation** (10 min) + +**5. Next Steps** (10 min) +" +}] +``` + +### 6. Link Resources +Connected agenda to CRM page, pricing sheet, and technical integration docs. + +## Outputs + +**Internal Pre-Read**: Full context for sales team +**Customer Agenda**: Professional meeting structure +**Both in Notion** with links to supporting materials + +## Key Success Factors +- Understood customer's specific pain points +- Researched industry context (fintech compliance) +- Mapped features to their needs +- Prepared competitive differentiators +- Structured demo around their use cases +- Pre-planned objection responses +- Clear next steps in agenda diff --git a/plugins/notion/skills/notion-meeting-intelligence/examples/executive-review.md b/plugins/notion/skills/notion-meeting-intelligence/examples/executive-review.md new file mode 100644 index 0000000..f5c328c --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/examples/executive-review.md @@ -0,0 +1,78 @@ +# Example: Executive Review Prep + +**User Request**: "Prep for quarterly executive review on Friday" + +## Workflow + +### 1. Search for Context +``` +Notion:search +query: "Q4 objectives" + "KPIs" + "quarterly results" +``` + +Found: +- Q4 OKRs and progress +- Product metrics dashboard +- Engineering velocity reports +- Customer feedback summary + +### 2. Fetch & Analyze +``` +Notion:fetch (5 pages) +``` + +**Key metrics**: +- **Revenue**: $2.4M ARR (96% of Q4 target) +- **Customer Growth**: 145 new customers (exceeds 120 target) +- **Churn**: 3.2% (below 5% target) +- **Product**: 3 major features shipped, 2 in beta +- **Engineering**: 94% uptime (above 95% SLA) + +### 3. Add Codex Research Context +Added context on: +- Industry benchmarks for SaaS metrics +- Typical Q4 sales patterns +- Best practices for executive presentations + +### 4. Create Pre-Read (Internal) +``` +Notion:notion-create-pages +title: "Q4 Review - Pre-Read (Internal)" +``` + +**Pre-read sections**: +- **Executive Summary**: Strong quarter, missed revenue by 4% but exceeded customer growth +- **Detailed Metrics**: All KPIs with trend lines +- **Wins**: Product launches, key customer acquisitions +- **Challenges**: Sales pipeline conversion, engineering hiring +- **Q1 Preview**: Strategic priorities + +### 5. Create Presentation Agenda +``` +Notion:notion-create-pages +title: "Q4 Executive Review - Agenda" +``` + +**Agenda** (90 min): +- Q4 Results Overview (15 min) +- Revenue & Growth Deep Dive (20 min) +- Product & Engineering Update (20 min) +- Customer Success Highlights (15 min) +- Q1 Strategic Plan (15 min) +- Discussion & Questions (15 min) + +### 6. Link Supporting Docs +Connected to OKRs, metrics dashboards, and Q1 planning docs. + +## Outputs + +**Internal Pre-Read**: Comprehensive context with honest assessment +**Executive Agenda**: Structured 90-min presentation +**Both in Notion** with links to supporting data + +## Key Success Factors +- Synthesized data from multiple sources (OKRs, metrics, feedback) +- Added industry context and benchmarks +- Created honest internal assessment (not just wins) +- Structured agenda with time allocations +- Linked to source data for drill-down during Q&A diff --git a/plugins/notion/skills/notion-meeting-intelligence/examples/project-decision.md b/plugins/notion/skills/notion-meeting-intelligence/examples/project-decision.md new file mode 100644 index 0000000..9e29062 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/examples/project-decision.md @@ -0,0 +1,431 @@ +# Example: Project Decision Meeting Prep + +## User Request + +> "Prep for tomorrow's architecture meeting where we need to decide on database migration strategy" + +## Workflow Execution + +### Step 1: Search for Context + +``` +Notion:search +query: "database migration" +query_type: "internal" +teamspace_id: "engineering-teamspace-id" +filters: { + created_date_range: { + start_date: "2024-01-01" + } +} +``` + +**Results Found:** +- "Database Migration Proposal" (Engineering) +- "Current Database Performance Issues" (Engineering) +- "Migration Options Analysis" (Architecture) +- "Previous Migration Post-mortem" (Engineering) + +### Step 2: Fetch Relevant Pages + +``` +Notion:fetch +id: "database-migration-proposal-page-id" +``` + +**Extracted**: Proposes migrating from MongoDB to PostgreSQL, outlines timeline and risks + +``` +Notion:fetch +id: "database-performance-issues-page-id" +``` + +**Extracted**: Current pain points - slow queries, schema flexibility issues, transaction limitations + +``` +Notion:fetch +id: "migration-options-analysis-page-id" +``` + +**Extracted**: Three options analyzed: +1. Stay on MongoDB with optimizations +2. Migrate to PostgreSQL +3. Hybrid approach (MongoDB + PostgreSQL) + +``` +Notion:fetch +id: "previous-migration-postmortem-page-id" +``` + +**Extracted**: Lessons from 2023 Redis migration - importance of rollback plan, testing strategy + +### Step 3: Synthesize Pre-Read + +**Key Information Identified:** +- **Problem**: MongoDB performance degrading, schema flexibility causing data consistency issues +- **Options**: Three approaches with different trade-offs +- **Recommendation**: PostgreSQL migration favored by engineering team +- **Concerns**: Downtime requirements, data migration complexity, application changes needed +- **Historical context**: Previous migration took 6 weeks, had rollback issues + +### Step 4: Create Meeting Agenda + +``` +Notion:notion-create-pages + +parent: { page_id: "meetings-parent-id" } + +pages: [{ + properties: { + "title": "Architecture Decision: Database Migration Strategy - Oct 15, 2025" + }, + content: "[Decision meeting agenda]" +}] +``` + +**Agenda Created:** + +```markdown +# Architecture Decision: Database Migration Strategy + +## Meeting Details +**Date & Time**: October 15, 2025 at 2:00 PM +**Duration**: 90 minutes +**Location**: Conference Room B / Zoom +**Facilitator**: Sarah Chen (Engineering Lead) +**Attendees**: +- Sarah Chen (Engineering Lead) +- David Kim (CTO) +- Alice Wang (Backend Team Lead) +- Bob Martinez (DevOps Lead) +- Carol Johnson (Product Director) + +## Pre-Read Summary + +### Background + +Our MongoDB database is experiencing performance degradation as we scale. Query response times have increased from 50ms to 400ms average over the past 6 months. The schema-less nature that initially provided flexibility is now causing data consistency issues and making application logic more complex. + +**Source**: Current Database Performance Issues + +### Current Situation + +**Performance metrics**: +- Average query time: 400ms (was 50ms 6 months ago) +- p95 query time: 1.2s (was 200ms) +- Database size: 500GB (growing 20GB/month) +- Connection pool exhaustion during peak traffic + +**Technical debt**: +- 15+ application-layer validation rules compensating for lack of schema +- Complex data migration scripts for schema changes +- Limited transaction support causing race conditions + +**Source**: Database Migration Proposal + +### Historical Context + +We successfully migrated from Redis to Memcached in 2023, which took 6 weeks. Key learnings: +- Underestimated application code changes (3 weeks instead of 1 week) +- Rollback plan was crucial when we discovered compatibility issues +- Parallel running period (dual writes) was essential for safe migration + +**Source**: Previous Migration Post-mortem + +## Decision Required + +**Question**: Which database migration strategy should we adopt? + +**Timeline**: Need decision by end of week to include in Q4 planning + +**Impact**: +- Engineering team (4-8 weeks of work) +- Application architecture +- Operations & monitoring +- Future feature development velocity + +## Options Analysis + +### Option A: Stay on MongoDB with Optimizations + +**Description**: Invest in MongoDB performance tuning, add indexes, upgrade to latest version, implement better query patterns. + +**Pros**: +- ✅ No migration complexity +- ✅ Team familiar with MongoDB +- ✅ Can implement immediately +- ✅ Lower risk +- ✅ Estimated 2 weeks effort + +**Cons**: +- ❌ Doesn't solve fundamental schema flexibility issues +- ❌ Still limited transaction support +- ❌ Performance improvements may be temporary +- ❌ Continues technical debt accumulation + +**Cost/Effort**: 2 weeks engineering + $5K/year additional MongoDB infrastructure + +**Risk**: Medium - Improvements may not be sufficient + +**Source**: Migration Options Analysis + +### Option B: Migrate to PostgreSQL + +**Description**: Full migration from MongoDB to PostgreSQL. Redesign schema with proper constraints, implement dual-write period, then cut over. + +**Pros**: +- ✅ Solves schema consistency issues +- ✅ Full ACID transactions +- ✅ Better performance for relational queries +- ✅ Lower long-term complexity +- ✅ Industry standard, easier hiring + +**Cons**: +- ❌ High migration effort (6-8 weeks) +- ❌ Requires schema redesign +- ❌ Application code changes extensive +- ❌ Risk of data loss during migration +- ❌ Downtime required (4-6 hours estimated) + +**Cost/Effort**: 8 weeks engineering + $8K migration costs - $15K/year MongoDB savings = net $7K/year savings + +**Risk**: High - Complex migration, application changes required + +**Recommendation**: ✅ **Favored by engineering team** + +**Source**: Database Migration Proposal + +### Option C: Hybrid Approach + +**Description**: Keep MongoDB for document-heavy data (logs, analytics), migrate transactional data to PostgreSQL. Run both databases. + +**Pros**: +- ✅ Phased migration (lower risk) +- ✅ Use best tool for each data type +- ✅ Can migrate incrementally +- ✅ Smaller initial scope (4 weeks) + +**Cons**: +- ❌ Increased operational complexity +- ❌ Two databases to maintain +- ❌ Data consistency between databases challenging +- ❌ Higher infrastructure costs +- ❌ Complex application logic + +**Cost/Effort**: 4 weeks initial + ongoing complexity + $10K/year additional infrastructure + +**Risk**: Medium - Operational complexity increases + +**Source**: Migration Options Analysis + +### Option D: Do Nothing + +**Description**: Accept current performance and continue with MongoDB as-is. + +**Implications**: +- Performance continues to degrade +- Technical debt increases +- Feature development slows +- Customer experience suffers +- Eventually forced into emergency migration + +**Not recommended** + +## Discussion Topics + +### Technical Feasibility +1. Can we achieve < 4 hours downtime for Option B? +2. What's the rollback plan if PostgreSQL migration fails? +3. How do we handle data migration for 500GB? +4. Schema design - what constraints do we need? + +### Business Impact +5. What's the customer impact of 4-6 hours downtime? +6. Can we schedule migration during low-traffic period? +7. How does this affect Q4 feature roadmap? +8. Cost-benefit analysis over 2-year horizon? + +### Risk Management +9. What are the biggest risks with Option B? +10. How do we test thoroughly before cutover? +11. What's the rollback procedure and time? +12. Do we have necessary expertise on team? + +### Timeline & Resources +13. Can we allocate 2 engineers full-time for 8 weeks? +14. Do we need external consultants? +15. What's the impact on other Q4 projects? +16. When could we realistically complete this? + +## Decision Framework + +**Evaluation criteria**: +1. **Performance improvement**: Will this solve our performance issues? +2. **Technical debt**: Does this reduce or increase complexity? +3. **Risk**: What's the probability and impact of failure? +4. **Cost**: Total cost of ownership (effort + infrastructure) +5. **Timeline**: Can we complete in Q4? +6. **Reversibility**: Can we roll back if needed? + +**Decision makers**: +- Primary: David Kim (CTO) +- Consulted: Sarah Chen (Engineering), Bob Martinez (DevOps) +- Informed: Product, Executive team + +## Decision + +*[To be filled during meeting]* + +**Selected Option**: ___________________ + +**Rationale**: + +**Timeline**: + +**Owner**: ___________________ + +**Success Criteria**: + +## Action Items + +*[To be filled during meeting]* + +- [ ] [Action item] - @[Owner] - Due: [Date] +- [ ] [Action item] - @[Owner] - Due: [Date] + +## Next Steps + +**Immediate**: +- Document decision rationale +- Create detailed implementation plan +- Schedule kickoff meeting + +**Follow-up meeting**: [Date] to review migration plan + +## Related Resources + +### Primary Sources +- Database Migration Proposal +- Current Database Performance Issues +- Migration Options Analysis +- Previous Migration Post-mortem + +### Supporting Documents +- MongoDB performance metrics dashboard +- PostgreSQL evaluation results +- Cost analysis spreadsheet +``` + +### Step 5: Share Agenda + +``` +Notion:notion-create-comment +parent: { page_id: "agenda-page-id" } +rich_text: [{ + type: "text", + text: { content: "📋 Meeting agenda ready! Please review the pre-read before tomorrow's meeting. cc: @Sarah @David @Alice @Bob @Carol" } +}] +``` + +## Summary Provided to User + +```markdown +## Meeting Prep Complete ✅ + +### Meeting Details +- **Topic**: Database Migration Strategy Decision +- **Date**: October 15, 2025 at 2:00 PM +- **Duration**: 90 minutes +- **Attendees**: 5 (Engineering leads + CTO + Product) + +### Agenda Created +Architecture Decision: Database Migration Strategy + +### Pre-Read Summary + +**Problem**: MongoDB performance degrading (50ms → 400ms queries), schema flexibility causing issues + +**Decision needed**: Choose between: +1. Stay on MongoDB (optimizations) - 2 weeks, low risk +2. Migrate to PostgreSQL - 8 weeks, high value, high risk ✅ Recommended +3. Hybrid approach - 4 weeks, medium complexity +4. Do nothing - Not recommended + +**Recommendation**: Engineering team favors PostgreSQL migration + +### Key Information Included + +✅ **Background context** from 4 relevant pages +✅ **Options analysis** with pros/cons for each +✅ **Cost-benefit** breakdown +✅ **Risk assessment** for each option +✅ **Discussion topics** (16 questions organized by theme) +✅ **Decision framework** with evaluation criteria +✅ **Historical context** from previous migration +✅ **Empty sections** for decision and action items + +### Discussion Topics Prepared + +**Technical** (4 questions): +- Downtime achievability +- Rollback planning +- Data migration approach +- Schema design + +**Business** (4 questions): +- Customer impact +- Scheduling +- Roadmap impact +- Cost analysis + +**Risk** (4 questions): +- Risk identification +- Testing strategy +- Rollback procedure +- Expertise availability + +**Resources** (4 questions): +- Team allocation +- External help needs +- Project trade-offs +- Timeline realism + +### Success Factors + +1. **Comprehensive research**: 4 relevant pages analyzed +2. **Clear options**: Pros/cons for each approach +3. **Structured discussion**: 16 questions organized by theme +4. **Decision framework**: Clear criteria for evaluation +5. **Historical context**: Lessons from previous migration +6. **Ready for outcomes**: Sections prepared for decision and actions + +**Meeting is well-prepared for productive decision-making! 📊** +``` + +## Key Features Demonstrated + +### Context Gathering +- **Targeted search**: Engineering teamspace, recent documents +- **Multiple perspectives**: Performance data, proposals, analysis, historical lessons +- **Comprehensive synthesis**: Combined technical, business, and historical context + +### Options Analysis +- **Structured comparison**: Pros/cons for each option +- **Cost-benefit**: Effort and infrastructure costs included +- **Risk assessment**: Probability and impact noted +- **Recommendation**: Clear engineering preference stated + +### Decision Support +- **Discussion topics**: 16 questions organized by theme +- **Decision framework**: Evaluation criteria defined +- **Decision makers**: Roles and responsibilities clear +- **Outcome capture**: Sections ready for decision and actions + +### Meeting Structure +- **Pre-read**: Comprehensive background (can be read in 10 minutes) +- **Options**: Clear comparison for quick decision +- **Discussion**: Structured topics prevent rambling +- **Capture**: Templates for decision and actions + +Perfect for: Architecture decisions, technical trade-offs, strategic choices + diff --git a/plugins/notion/skills/notion-meeting-intelligence/examples/sprint-planning.md b/plugins/notion/skills/notion-meeting-intelligence/examples/sprint-planning.md new file mode 100644 index 0000000..2c7485e --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/examples/sprint-planning.md @@ -0,0 +1,80 @@ +# Example: Sprint Planning Meeting Prep + +**User Request**: "Prepare for tomorrow's sprint planning meeting" + +## Workflow + +### 1. Search for Context +``` +Notion:search +query: "sprint planning" + "product backlog" +teamspace_id: "engineering-team" +``` + +Found: +- Last sprint retrospective +- Product backlog (prioritized) +- Current sprint progress +- Team capacity notes + +### 2. Fetch Details +``` +Notion:fetch (4 pages) +``` + +**Key context**: +- **Last Sprint**: Completed 32/35 story points (91%) +- **Velocity**: Consistent 30-35 points over last 3 sprints +- **Team**: 5 engineers, 1 on vacation next sprint (80% capacity) +- **Top Backlog Items**: User auth improvements, API performance, mobile responsive fixes + +### 3. Query Current Sprint Tasks +``` +Notion:notion-query-data-sources +query: "SELECT * FROM tasks WHERE Sprint = 'Sprint 24' AND Status != 'Done'" +``` + +3 tasks carrying over (technical debt items) + +### 4. Create Pre-Read (Internal) +``` +Notion:notion-create-pages +title: "Sprint 25 Planning - Pre-Read (Internal)" +``` + +**Pre-read included**: +- Sprint 24 summary (velocity, what carried over) +- Team capacity for Sprint 25 +- Top backlog candidates with story points +- Technical dependencies +- Risk items (auth changes need QA time) + +### 5. Create Agenda +``` +Notion:notion-create-pages +title: "Sprint 25 Planning - Agenda" +``` + +**Agenda**: +- Review Sprint 24 completion (5 min) +- Discuss carryover items (5 min) +- Review capacity (28 points available) +- Select backlog items (30 min) +- Identify dependencies & risks (10 min) +- Confirm commitments (10 min) + +### 6. Link Documents +Cross-linked pre-read and agenda, referenced last retro and backlog. + +## Output Summary + +**Internal Pre-Read**: Team context, capacity, blockers +**External Agenda**: Meeting structure, discussion topics +**Both saved to Notion** and linked to project pages + +## Key Success Factors +- Gathered sprint history for velocity trends +- Calculated realistic capacity (account for PTO) +- Identified carryover items upfront +- Pre-read gave team context before meeting +- Agenda kept meeting focused and timeboxed diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/brainstorming-template.md b/plugins/notion/skills/notion-meeting-intelligence/reference/brainstorming-template.md new file mode 100644 index 0000000..0e31334 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/brainstorming-template.md @@ -0,0 +1,81 @@ +# Brainstorming Meeting Template + +Use this template for creative ideation and brainstorming sessions. + +```markdown +# [Topic] Brainstorming - [Date] + +## Meeting Details +**Date**: [Date] +**Facilitator**: [Name] +**Note-taker**: [Name] +**Attendees**: [List] + +## Objective + +[Clear statement of what we're brainstorming] + +**Success looks like**: [How we'll know brainstorming was successful] + +## Background & Context + +[Context from research - 2-3 paragraphs] + +**Related Pages**: +- Context Page 1 +- Context Page 2 + +## Constraints + +- [Constraint] +- [Constraint] +- [Constraint] + +## Seed Ideas + +[Starting ideas from research to spark discussion]: + +1. **[Idea]**: [Brief description] +2. **[Idea]**: [Brief description] + +## Ground Rules + +- No criticism during ideation +- Build on others' ideas +- Quantity over quality initially +- Wild ideas welcome + +## Brainstorming Notes + +### Ideas Generated + +[To be filled during meeting] + +1. [Idea with brief description] +2. [Idea with brief description] + +### Themes/Patterns + +[Groupings that emerge] + +## Evaluation + +[If time permits, evaluate top ideas] + +### Top Ideas + +| Idea | Feasibility | Impact | Effort | Score | +|------|-------------|---------|--------|-------| +| [Idea] | [H/M/L] | [H/M/L] | [H/M/L] | [#] | + +## Next Steps + +- [ ] [Action to explore idea] +- [ ] [Action to prototype] +- [ ] [Action to research] + +## Follow-up + +**Next meeting**: [Date to reconvene] +``` + diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/decision-meeting-template.md b/plugins/notion/skills/notion-meeting-intelligence/reference/decision-meeting-template.md new file mode 100644 index 0000000..5c0fa56 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/decision-meeting-template.md @@ -0,0 +1,94 @@ +# Decision Meeting Template + +Use this template when you need to make an important decision with your team. + +```markdown +# [Decision Topic] - [Date] + +## Meeting Details +**Date & Time**: [Date and time] +**Duration**: [Length] +**Attendees**: [List of attendees with roles] +**Location**: [Physical location or video link] +**Facilitator**: [Name] + +## Pre-Read Summary + +### Background +[2-3 sentences providing context from related project pages] + +**Related Pages**: +- Project Overview +- Previous Discussion + +### Current Situation +[What brings us to this decision point] + +## Decision Required + +**Question**: [Clear statement of decision needed] + +**Timeline**: [When decision needs to be made] + +**Impact**: [Who/what is affected by this decision] + +## Options Analysis + +### Option A: [Name] +**Description**: [What this option entails] + +**Pros**: +- [Advantage] +- [Advantage] + +**Cons**: +- [Disadvantage] +- [Disadvantage] + +**Cost/Effort**: [Estimate] +**Risk**: [Risk assessment] + +### Option B: [Name] +[Repeat structure] + +### Option C: Do Nothing +**Description**: What happens if we don't decide +**Implications**: [Consequences] + +## Recommendation + +[If there is a recommended option, state it with rationale] + +## Discussion Topics + +1. [Topic to discuss] +2. [Clarification needed on] +3. [Trade-offs to consider] + +## Decision Framework + +**Criteria for evaluation**: +- [Criterion 1] +- [Criterion 2] +- [Criterion 3] + +## Decision + +[To be filled during meeting] + +**Selected Option**: [Option chosen] +**Rationale**: [Why] +**Owner**: [Who will implement] +**Timeline**: [When] + +## Action Items + +- [ ] [Action] - @[Owner] - Due: [Date] +- [ ] [Action] - @[Owner] - Due: [Date] + +## Follow-up + +**Next review**: [Date] +**Success metrics**: [How we'll know this worked] +``` + diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/one-on-one-template.md b/plugins/notion/skills/notion-meeting-intelligence/reference/one-on-one-template.md new file mode 100644 index 0000000..9885beb --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/one-on-one-template.md @@ -0,0 +1,58 @@ +# 1:1 Meeting Template + +Use this template for manager/report one-on-one meetings. + +```markdown +# 1:1: [Manager] & [Report] - [Date] + +## Meeting Details +**Date**: [Date] +**Last meeting**: Previous 1:1 + +## Agenda + +### [Report]'s Topics +1. [Topic to discuss] +2. [Question or concern] + +### [Manager]'s Topics +1. [Topic to cover] +2. [Feedback or update] + +## Discussion Notes + +### [Topic 1] +[Discussion points] + +**Action items**: +- [ ] [Action] - @[Owner] + +### [Topic 2] +[Discussion points] + +## Career Development + +**Current focus**: [Development goal] +**Progress**: [Update on progress] + +## Feedback + +**What's going well**: +- [Positive feedback] + +**Areas for growth**: +- [Developmental feedback] + +## Action Items + +- [ ] [Action] - @[Report] - Due: [Date] +- [ ] [Action] - @[Manager] - Due: [Date] + +## Next Meeting + +**Date**: [Date] +**Topics to cover**: +- [Carry-over topic] +- [Upcoming topic] +``` + diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/retrospective-template.md b/plugins/notion/skills/notion-meeting-intelligence/reference/retrospective-template.md new file mode 100644 index 0000000..d3e3673 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/retrospective-template.md @@ -0,0 +1,58 @@ +# Retrospective Template + +Use this template for sprint retrospectives and team retrospectives. + +```markdown +# Sprint [#] Retrospective - [Date] + +## Meeting Details +**Date**: [Date] +**Team**: [Team] +**Sprint**: [Sprint dates] +**Facilitator**: [Name] + +## Sprint Summary + +**Sprint Goal**: [Goal] +**Goal Met**: Yes / Partially / No + +**Completed**: [#] points +**Velocity**: [#] points +**Planned**: [#] points + +## Pre-Read + +**Sprint Metrics**: +- Tasks completed: [#] +- Tasks carried over: [#] +- Bugs found: [#] +- Blockers encountered: [#] + +## Discussion + +### What Went Well (Keep) + +[Team input during meeting] + +### What Didn't Go Well (Stop) + +[Team input during meeting] + +### What To Try (Start) + +[Team input during meeting] + +### Shout-outs + +[Team recognition] + +## Action Items + +- [ ] [Improvement to implement] - @[Owner] - Due: [Date] +- [ ] [Process change] - @[Owner] - Due: [Date] + +## Follow-up + +**Review actions in**: [Next retro date] +``` + diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/sprint-planning-template.md b/plugins/notion/skills/notion-meeting-intelligence/reference/sprint-planning-template.md new file mode 100644 index 0000000..36adb3b --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/sprint-planning-template.md @@ -0,0 +1,68 @@ +# Sprint Planning Template + +Use this template for agile sprint planning meetings. + +```markdown +# Sprint [#] Planning - [Date] + +## Meeting Details +**Date**: [Date] +**Team**: [Team name] +**Sprint Duration**: [Dates] + +## Sprint Goal + +[Clear statement of what this sprint aims to accomplish] + +## Capacity + +| Team Member | Availability | Capacity (points) | +|-------------|--------------|-------------------| +| [Name] | [%] | [#] | +| **Total** | | [#] | + +## Backlog Review + +### High Priority Items + +[From product backlog, linked from task database] + +- Task 1 - [Points] +- Task 2 - [Points] + +## Sprint Backlog + +### Committed Items + +- [x] Task - [Points] - @[Owner] +- [ ] Task - [Points] - @[Owner] + +**Total committed**: [Points] + +### Stretch Goals + +- [ ] Task - [Points] + +## Dependencies & Risks + +**Dependencies**: +- [Dependency] + +**Risks**: +- [Risk] + +## Definition of Done + +- [ ] Code complete and reviewed +- [ ] Tests written and passing +- [ ] Documentation updated +- [ ] Deployed to staging +- [ ] QA approved + +## Next Steps + +- Team begins sprint work +- Daily standups at [Time] +- Sprint review on [Date] +``` + diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/status-update-template.md b/plugins/notion/skills/notion-meeting-intelligence/reference/status-update-template.md new file mode 100644 index 0000000..f5d8789 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/status-update-template.md @@ -0,0 +1,74 @@ +# Status Update Meeting Template + +Use this template for regular project status updates and check-ins. + +```markdown +# [Project Name] Status Update - [Date] + +## Meeting Details +**Date**: [Date and time] +**Attendees**: [List] +**Project**: Project Page + +## Executive Summary + +**Status**: 🟢 On Track / 🟡 At Risk / 🔴 Behind + +**Progress**: [Percentage] complete +**Timeline**: [Status vs original plan] + +## Progress Since Last Meeting + +### Completed +- [Accomplishment with specifics] +- [Accomplishment with specifics] + +### In Progress +- [Work item and status] +- [Work item and status] + +## Metrics + +| Metric | Current | Target | Status | +|--------|---------|--------|--------| +| [Metric] | [Value] | [Value] | [Icon] | +| [Metric] | [Value] | [Value] | [Icon] | + +## Upcoming Work + +**Next 2 Weeks**: +- [Planned work] +- [Planned work] + +**Next Month**: +- [Milestone or major work] + +## Blockers & Risks + +### Active Blockers +- **[Blocker]**: [Description and impact] + - Action: [What's being done] + +### Risks +- **[Risk]**: [Description] + - Mitigation: [Strategy] + +## Discussion Topics + +1. [Topic requiring input] +2. [Topic for alignment] + +## Decisions Needed + +- [Decision] or None + +## Action Items + +- [ ] [Action] - @[Owner] - Due: [Date] + +## Next Meeting + +**Date**: [Date] +**Focus**: [What next meeting will cover] +``` + diff --git a/plugins/notion/skills/notion-meeting-intelligence/reference/template-selection-guide.md b/plugins/notion/skills/notion-meeting-intelligence/reference/template-selection-guide.md new file mode 100644 index 0000000..d528977 --- /dev/null +++ b/plugins/notion/skills/notion-meeting-intelligence/reference/template-selection-guide.md @@ -0,0 +1,56 @@ +# Meeting Template Selection Guide + +Choose the right template for your meeting type. + +## Template Overview + +| Meeting Type | Use This Template | When to Use | +|--------------|-------------------|-------------| +| Make a decision | [Decision Meeting](decision-meeting-template.md) | Need to evaluate options and reach a decision | +| Project update | [Status Update](status-update-template.md) | Regular check-ins, progress reviews | +| Generate ideas | [Brainstorming](brainstorming-template.md) | Creative ideation, problem-solving | +| Sprint planning | [Sprint Planning](sprint-planning-template.md) | Planning agile sprint work | +| Sprint retro | [Retrospective](retrospective-template.md) | Reflecting on completed work | +| Manager/report | [1:1 Meeting](one-on-one-template.md) | Regular one-on-one check-ins | +| Weekly team sync | [Status Update](status-update-template.md) (simplified) | Routine team synchronization | + +## Quick Decision Tree + +``` +What's the primary purpose? + +├─ Make a decision +│ └─ Use: Decision Meeting Template +│ +├─ Update on progress +│ └─ Use: Status Update Template +│ +├─ Generate ideas +│ └─ Use: Brainstorming Template +│ +├─ Plan sprint work +│ └─ Use: Sprint Planning Template +│ +├─ Reflect on past work +│ └─ Use: Retrospective Template +│ +└─ Manager/report check-in + └─ Use: 1:1 Meeting Template +``` + +## Template Customization + +All templates can be customized: +- **Simplify** for shorter meetings +- **Add sections** for specific needs +- **Combine elements** from multiple templates +- **Adapt language** for your team culture + +## Best Practices + +1. **Choose template first**: Select before gathering context +2. **Gather Notion content**: Search and fetch relevant pages +3. **Enrich with research**: Add Codex insights where valuable +4. **Customize as needed**: Adapt template to specific situation +5. **Share early**: Give attendees time to review + diff --git a/plugins/notion/skills/notion-research-documentation/LICENSE.txt b/plugins/notion/skills/notion-research-documentation/LICENSE.txt new file mode 100644 index 0000000..0871708 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/LICENSE.txt @@ -0,0 +1,7 @@ +Copyright 2025 Notion Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/notion/skills/notion-research-documentation/SKILL.md b/plugins/notion/skills/notion-research-documentation/SKILL.md new file mode 100644 index 0000000..ed2e942 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/SKILL.md @@ -0,0 +1,59 @@ +--- +name: notion-research-documentation +description: Research across Notion and synthesize into structured documentation; use when gathering info from multiple Notion sources to produce briefs, comparisons, or reports with citations. +metadata: + short-description: Research Notion content and produce briefs/reports +--- + +# Research & Documentation + +Pull relevant Notion pages, synthesize findings, and publish clear briefs or reports (with citations and links to sources). + +## Quick start +1) Find sources with `Notion:search` using targeted queries; confirm scope with the user. +2) Fetch pages via `Notion:fetch`; note key sections and capture citations (`reference/citations.md`). +3) Choose output format (brief, summary, comparison, comprehensive report) using `reference/format-selection-guide.md`. +4) Draft in Notion with `Notion:notion-create-pages` using the matching template (quick, summary, comparison, comprehensive). +5) Link sources and add a references/citations section; update as new info arrives with `Notion:notion-update-page`. + +## Workflow +### 0) If any MCP call fails because Notion MCP is not connected, pause and set it up: +1. Add the Notion MCP: + - `codex mcp add notion --url https://mcp.notion.com/mcp` +2. Enable remote MCP client: + - Set `[features].rmcp_client = true` in `config.toml` **or** run `codex --enable rmcp_client` +3. Log in with OAuth: + - `codex mcp login notion` + +After successful login, the user will have to restart codex. You should finish your answer and tell them so when they try again they can continue with Step 1. + +### 1) Gather sources +- Search first (`Notion:search`); refine queries, and ask the user to confirm if multiple results appear. +- Fetch relevant pages (`Notion:fetch`), skim for facts, metrics, claims, constraints, and dates. +- Track each source URL/ID for later citation; prefer direct quotes for critical facts. + +### 2) Select the format +- Quick readout → quick brief. +- Single-topic dive → research summary. +- Option tradeoffs → comparison. +- Deep dive / exec-ready → comprehensive report. +- See `reference/format-selection-guide.md` for when to pick each. + +### 3) Synthesize +- Outline before writing; group findings by themes/questions. +- Note evidence with source IDs; flag gaps or contradictions. +- Keep user goal in view (decision, summary, plan, recommendation). + +### 4) Create the doc +- Pick the matching template in `reference/` (brief, summary, comparison, comprehensive) and adapt it. +- Create the page with `Notion:notion-create-pages`; include title, summary, key findings, supporting evidence, and recommendations/next steps when relevant. +- Add citations inline and a references section; link back to source pages. + +### 5) Finalize & handoff +- Add highlights, risks, and open questions. +- If the user needs follow-ups, create tasks or a checklist in the page; link any task database entries if applicable. +- Share a short changelog or status using `Notion:notion-update-page` when updating. + +## References and examples +- `reference/` — search tactics, format selection, templates, and citation rules (e.g., `advanced-search.md`, `format-selection-guide.md`, `research-summary-template.md`, `comparison-template.md`, `citations.md`). +- `examples/` — end-to-end walkthroughs (e.g., `competitor-analysis.md`, `technical-investigation.md`, `market-research.md`, `trip-planning.md`). diff --git a/plugins/notion/skills/notion-research-documentation/agents/openai.yaml b/plugins/notion/skills/notion-research-documentation/agents/openai.yaml new file mode 100644 index 0000000..8ec6a73 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/agents/openai.yaml @@ -0,0 +1,14 @@ +interface: + display_name: "Notion Research & Documentation" + short_description: "Research Notion content and produce briefs/reports" + icon_small: "./assets/notion-small.svg" + icon_large: "./assets/notion.png" + default_prompt: "Research this topic in Notion and produce a sourced brief with clear recommendations." + +dependencies: + tools: + - type: "mcp" + value: "notion" + description: "Notion MCP server" + transport: "streamable_http" + url: "https://mcp.notion.com/mcp" diff --git a/plugins/notion/skills/notion-research-documentation/assets/notion-small.svg b/plugins/notion/skills/notion-research-documentation/assets/notion-small.svg new file mode 100644 index 0000000..455d4be --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/assets/notion-small.svg @@ -0,0 +1,11 @@ + diff --git a/plugins/notion/skills/notion-research-documentation/assets/notion.png b/plugins/notion/skills/notion-research-documentation/assets/notion.png new file mode 100644 index 0000000..085023c Binary files /dev/null and b/plugins/notion/skills/notion-research-documentation/assets/notion.png differ diff --git a/plugins/notion/skills/notion-research-documentation/evaluations/README.md b/plugins/notion/skills/notion-research-documentation/evaluations/README.md new file mode 100644 index 0000000..08ddcdd --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/evaluations/README.md @@ -0,0 +1,109 @@ +# Research & Documentation Skill Evaluations + +Evaluation scenarios for testing the Research & Documentation skill across different Codex models. + +## Purpose + +These evaluations ensure the Research & Documentation skill: +- Searches across Notion workspace effectively +- Synthesizes information from multiple sources +- Selects appropriate research report format +- Creates comprehensive documentation with proper citations +- Works consistently across Haiku, Sonnet, and Opus + +## Evaluation Files + +### basic-research.json +Tests basic research workflow with synthesis across multiple Notion pages. + +**Scenario**: Research Q4 product roadmap and create summary +**Key Behaviors**: +- Searches Notion for roadmap-related pages +- Fetches multiple relevant pages (roadmap, product docs, meeting notes) +- Synthesizes information from different sources +- Selects appropriate format (Research Summary) +- Includes citations linking back to source pages +- Creates structured document with clear sections + +### research-to-database.json +Tests creating research documentation in a Notion database with properties. + +**Scenario**: Research competitor landscape and save to Research database +**Key Behaviors**: +- Searches for existing competitive intelligence in Notion +- Identifies Research database as target +- Fetches database schema to understand properties +- Creates page with correct property values (Research Type, Status, Date, etc.) +- Structures content with comparison format +- Includes source citations for both Notion pages and external research + +## Running Evaluations + +1. Enable the `research-documentation` skill +2. Submit the query from the evaluation file +3. Verify the skill searches Notion workspace comprehensively +4. Check that multiple source pages are fetched and synthesized +5. Verify appropriate format is selected (Research Summary, Comprehensive Report, Quick Brief, Comparison) +6. Confirm citations link back to sources +7. Test with Haiku, Sonnet, and Opus + +## Expected Skill Behaviors + +Research & Documentation evaluations should verify: + +### Notion Search & Synthesis +- Searches workspace with relevant queries +- Fetches multiple source pages (3-5+) +- Synthesizes information across sources +- Identifies patterns and insights +- Handles conflicting information appropriately + +### Format Selection +- Chooses correct format based on scope and depth: + - **Research Summary**: Quick overview with key findings + - **Comprehensive Report**: Deep analysis with multiple sections + - **Quick Brief**: Fast facts and takeaways + - **Comparison**: Side-by-side analysis +- Applies format structure consistently +- Uses appropriate sections and headings + +### Citation & Attribution +- Includes citations for all Notion sources +- Uses mention-page tags: `` +- Attributes findings to specific sources +- Distinguishes between Notion content and Codex research +- Links related documents + +### Document Quality +- Title clearly indicates research topic and date +- Executive summary or key findings upfront +- Organized with clear hierarchy +- Actionable insights and recommendations +- Appropriate depth for the query + +## Creating New Evaluations + +When adding Research & Documentation evaluations: + +1. **Test different research types** - Product research, competitive analysis, technical investigation, market research +2. **Vary source count** - Synthesis of 2-3 pages vs. 10+ pages +3. **Test format selection** - Does it choose the right format for the scope? +4. **Include database targets** - Not just standalone pages +5. **Test citation accuracy** - Are all sources properly attributed? +6. **Cross-workspace search** - Testing search across teamspaces if applicable + +## Example Success Criteria + +**Good** (specific, testable): +- "Searches Notion for 'roadmap' and 'Q4' and 'product'" +- "Fetches at least 3 different source pages" +- "Includes citation for each key finding using mention-page tags" +- "Creates page with title format 'Research: [Topic] - [Date]'" +- "Uses Research Summary format with sections: Executive Summary → Key Findings → Details → Recommendations → Sources" + +**Bad** (vague, untestable): +- "Searches Notion effectively" +- "Creates comprehensive research" +- "Uses sources appropriately" +- "Good documentation" + diff --git a/plugins/notion/skills/notion-research-documentation/evaluations/basic-research.json b/plugins/notion/skills/notion-research-documentation/evaluations/basic-research.json new file mode 100644 index 0000000..517e914 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/evaluations/basic-research.json @@ -0,0 +1,28 @@ +{ + "name": "Basic Research and Documentation", + "skills": ["research-documentation"], + "query": "Research our API authentication approach and create a summary document in Notion", + "expected_behavior": [ + "Searches Notion workspace for authentication-related pages using Notion:search", + "Uses appropriate search terms like 'API authentication', 'auth', 'API security'", + "Applies filters if relevant (e.g., created_date_range, creator filters)", + "Fetches at least 2-3 relevant pages using Notion:fetch to get detailed content", + "Analyzes the fetched content to extract key information about authentication approach", + "Creates a structured research summary document using Research Summary format (see reference/formats.md)", + "Includes sections: Executive Summary, Key Findings, Detailed Analysis, Recommendations, Sources", + "Cites source pages using tags for proper linking", + "Uses Notion:notion-create-pages to save the document to Notion", + "Applies Notion-flavored markdown with headings, bullets, and clear structure", + "Places document appropriately (asks user or uses project/research area)" + ], + "success_criteria": [ + "Document contains synthesized information from multiple searched pages", + "At least 2-3 source pages are cited with mention-page tags", + "Document follows Research Summary format structure from reference/formats.md", + "Title is descriptive with topic and date (e.g., 'API Authentication Research - Oct 2025')", + "Content is concise but comprehensive with clear findings", + "Uses Notion markdown correctly (headings, lists, mentions)", + "Document is placed in appropriate location or user is consulted" + ] +} + diff --git a/plugins/notion/skills/notion-research-documentation/evaluations/research-to-database.json b/plugins/notion/skills/notion-research-documentation/evaluations/research-to-database.json new file mode 100644 index 0000000..3bc09bf --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/evaluations/research-to-database.json @@ -0,0 +1,29 @@ +{ + "name": "Research with Database Integration", + "skills": ["research-documentation"], + "query": "Research competitor pricing strategies and add to our Research database", + "expected_behavior": [ + "Searches for competitor and pricing information using Notion:search", + "Applies appropriate search strategy (see reference/advanced-search.md patterns)", + "Fetches relevant pages using Notion:fetch and synthesizes findings", + "Recognizes need to add to database (mentioned in query)", + "Searches for or asks about the Research database location", + "Fetches database using Notion:fetch to get schema, data sources, and properties", + "Identifies correct data source from tags if multiple exist", + "Creates page with appropriate database properties (Type: Competitor Analysis, Category, Tags, Date, Status, etc.)", + "Uses parent: { data_source_id: 'collection://...' } for correct database placement", + "Includes research content using Competitor Analysis format (see reference/formats.md)", + "Sets all required properties from schema with correct values", + "Cites sources using mention-page tags per reference/citations.md" + ], + "success_criteria": [ + "Page is created in correct database using data_source_id as parent", + "All required database properties are set correctly", + "Property values match available options from fetched schema", + "Content follows Competitor Analysis format structure", + "Sources are cited with proper mention-page tags", + "Title is descriptive (e.g., 'Competitor Pricing Analysis')", + "Research synthesizes findings rather than just listing sources" + ] +} + diff --git a/plugins/notion/skills/notion-research-documentation/examples/competitor-analysis.md b/plugins/notion/skills/notion-research-documentation/examples/competitor-analysis.md new file mode 100644 index 0000000..65167b6 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/examples/competitor-analysis.md @@ -0,0 +1,283 @@ +# Example: Competitor Analysis + +## User Request + +> "Research competitor pricing models and create a comparison document" + +## Workflow Execution + +### Step 1: Search for Competitor Information + +``` +Notion:search +query: "competitor pricing" +query_type: "internal" +filters: { + created_date_range: { + start_date: "2024-01-01" + } +} +``` + +**Results Found:** +- "Competitor Analysis - Stripe" (Product) +- "Market Research Q3 2024" (Strategy) +- "Pricing Strategy Discussion" (Product) +- "Competitor Feature Matrix" (Product) + +### Step 2: Fetch and Analyze + +``` +Notion:fetch for each page +``` + +**Extracted Information:** + +**Stripe**: +- Pay-as-you-go: 2.9% + $0.30 per transaction +- No monthly fees +- Volume discounts available + +**Square**: +- Pay-as-you-go: 2.6% + $0.10 per transaction +- Monthly plan: $29/month with lower rates +- Hardware costs separate + +**PayPal**: +- Standard: 3.49% + $0.49 per transaction +- Advanced: $30/month + 2.59% + $0.49 +- Multi-currency fees + +**Adyen**: +- Interchange++ pricing model +- Custom enterprise pricing +- Setup and monthly fees apply + +### Step 3: Create Comparison Document + +``` +Notion:notion-create-pages + +pages: [{ + properties: { + "title": "Competitor Pricing Analysis - Payment Processors" + }, + content: "[Comparison document using template]" +}] +``` + +## Output Document + +```markdown +# Competitor Pricing Analysis - Payment Processors + +**Date**: October 14, 2025 +**Research Team**: Product Strategy + +## Executive Summary + +We analyzed four major payment processor competitors: Stripe, Square, PayPal, and Adyen. Pricing models vary from simple pay-as-you-go (Stripe) to complex interchange-plus (Adyen). Key insight: All competitors offer volume discounts for high-transaction merchants, with breakpoints typically at $100K/month processing volume. + +## Comparison Matrix + +| Feature | Stripe | Square | PayPal | Adyen | +|---------|--------|--------|--------|-------| +| **Base Rate** | 2.9% + $0.30 | 2.6% + $0.10 | 3.49% + $0.49 | Interchange++ | +| **Monthly Fee** | $0 | $0-29 | $0-30 | Custom | +| **Volume Discounts** | Yes, >$80K | Yes, >$250K | Yes, >$100K | Yes, custom | +| **Setup Fee** | $0 | $0 | $0 | $1,000-5,000 | +| **Multi-currency** | 1% extra | 3% extra | 3-4% extra | Included | +| **Chargeback Fee** | $15 | $15-25 | $20 | Custom | +| **Target Market** | Startups-Enterprise | Small-Medium | Small-Medium | Enterprise | + +## Detailed Analysis + +### Stripe + +**Pricing Structure**: +- **Standard**: 2.9% + $0.30 per successful card charge +- **Volume discounts**: Available for businesses processing >$80,000/month +- **International cards**: +1% fee +- **Currency conversion**: 1% above market rate + +**Strengths**: +- Simple, transparent pricing +- No setup fees or monthly minimums +- Excellent developer experience +- Quick onboarding + +**Weaknesses**: +- Higher per-transaction fee for high volume +- Volume discounts less aggressive than Adyen + +**Best for**: Startups and growth-stage companies needing quick integration + +**Source**: Competitor Analysis - Stripe + +### Square + +**Pricing Structure**: +- **Pay-as-you-go**: 2.6% + $0.10 per tap, dip, or swipe +- **Keyed-in**: 3.5% + $0.15 +- **Plus plan**: $29/month for lower rates (2.5% + $0.10) +- **Premium plan**: Custom pricing + +**Strengths**: +- Lowest per-transaction fee for in-person +- All-in-one hardware + software +- No long-term contracts + +**Weaknesses**: +- Higher rates for online/keyed transactions +- Hardware costs ($49-$299) +- Less suitable for online-only businesses + +**Best for**: Brick-and-mortar retail and restaurants + +**Source**: Market Research Q3 2024 + +### PayPal + +**Pricing Structure**: +- **Standard**: 3.49% + $0.49 per transaction +- **Advanced**: $30/month + 2.59% + $0.49 +- **Payments Pro**: Additional $30/month for direct credit card processing + +**Strengths**: +- Huge customer base (PayPal checkout) +- Buyer protection increases trust +- International reach (200+ countries) + +**Weaknesses**: +- Highest per-transaction fees +- Complex fee structure +- Account holds and reserves common + +**Best for**: Businesses where PayPal brand trust matters (e-commerce, marketplaces) + +**Source**: Pricing Strategy Discussion + +### Adyen + +**Pricing Structure**: +- **Interchange++**: Actual interchange + scheme fees + fixed markup +- **Setup fee**: $1,000-5,000 (negotiable) +- **Monthly minimum**: Typically $10,000+ processing volume +- **Per-transaction**: Interchange + 0.6% + $0.12 (example) + +**Strengths**: +- Most transparent cost structure at scale +- Lowest effective rate for high volume +- True multi-currency (100+ currencies) +- Direct connections to schemes + +**Weaknesses**: +- Complex pricing requires analysis +- High minimums ($10K+/month) +- Longer integration time +- Not suitable for small businesses + +**Best for**: Enterprise with $1M+/month processing volume + +**Source**: Competitor Feature Matrix + +## Pricing Trends & Insights + +### Volume-Based Discounting +All competitors offer discounts at scale: +- **Entry point**: $80K-$250K/month processing +- **Typical discount**: 10-30 basis points reduction +- **Negotiation leverage**: Begins at $500K/month+ + +### Hidden Costs to Consider + +| Cost Type | Stripe | Square | PayPal | Adyen | +|-----------|--------|--------|--------|-------| +| Chargeback | $15 | $15-25 | $20 | $15-25 | +| Account verification | $0 | $0 | $0 | Varies | +| PCI compliance | $0 | $0 | $0 | $0 | +| Currency conversion | 1% | 3% | 3-4% | 0% | +| Refund fees | Returned | Returned | Not returned | Varies | + +### Market Positioning + +``` +High Volume / Enterprise + ↑ + | Adyen + | + | Stripe + | + | Square PayPal + | + └──────────────────→ + Small / Simple Complex / International +``` + +## Strategic Implications + +### For Startups (<$100K/month) +**Recommended**: Stripe +- Lowest friction to start +- No upfront costs +- Great documentation +- Acceptable rates at this scale + +### For Growing Companies ($100K-$1M/month) +**Recommended**: Stripe or Square +- Negotiate volume discounts +- Evaluate interchange++ if international +- Consider Square if in-person dominant + +### For Enterprises (>$1M/month) +**Recommended**: Adyen or Negotiated Stripe +- Interchange++ models save significantly +- Direct scheme connections +- Multi-region capabilities matter +- ROI on integration complexity + +## Recommendations + +1. **Immediate**: Benchmark our current 2.8% + $0.25 against Stripe's standard +2. **Short-term**: Request volume discount quote from Stripe at our current $150K/month +3. **Long-term**: Evaluate Adyen when we cross $1M/month threshold + +## Next Steps + +- [ ] Request detailed pricing proposal from Stripe for volume discounts +- [ ] Create pricing calculator comparing all 4 at different volume levels +- [ ] Interview customers about payment method preferences +- [ ] Analyze our transaction mix (domestic vs international, card types) + +## Sources + +### Primary Research +- Competitor Analysis - Stripe +- Market Research Q3 2024 +- Pricing Strategy Discussion +- Competitor Feature Matrix + +### External References +- Stripe.com pricing page (Oct 2025) +- Square pricing documentation +- PayPal merchant fees +- Adyen pricing transparency report +``` + +## Key Success Factors + +1. **Structured comparison**: Matrix format for quick scanning +2. **Multiple dimensions**: Price, features, target market +3. **Strategic recommendations**: Not just data, but implications +4. **Visual elements**: Table and positioning diagram +5. **Actionable next steps**: Clear recommendations +6. **Comprehensive sources**: Internal research + external validation + +## Workflow Pattern Demonstrated + +- **Date-filtered search** (recent information only) +- **Multiple competitor synthesis** (4 different companies) +- **Comparison template** (matrix + detailed analysis) +- **Strategic layer** (implications and recommendations) +- **Action-oriented** (next steps included) + diff --git a/plugins/notion/skills/notion-research-documentation/examples/market-research.md b/plugins/notion/skills/notion-research-documentation/examples/market-research.md new file mode 100644 index 0000000..5309ca5 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/examples/market-research.md @@ -0,0 +1,62 @@ +# Example: Market Research + +**User Request**: "Research the current state of AI coding assistants market and create a summary document in Notion" + +## Workflow + +### 1. Search +``` +Notion:search +query: "AI coding assistants market" +``` +Found 3 relevant pages across Engineering, Strategy, and Product teamspaces. + +### 2. Fetch & Analyze +``` +Notion:fetch (3x) +``` +Extracted market size, competitive landscape, technology trends from source pages. + +### 3. Create Documentation +``` +Notion:notion-create-pages +``` + +## Output (Condensed) + +```markdown +# AI Coding Assistants Market Research - Oct 2025 + +## Executive Summary +The AI coding assistant market is experiencing 150%+ YoY growth. GitHub Copilot dominates with 60% share, but specialized tools are gaining traction in specific niches. + +## Key Findings + +### Market Size and Growth +$800M in 2024 → $2.5B projected by 2026. Developer adoption: 23% (2023) → 47% (2024). +Source: Market Trends Q3 2025 + +### Competitive Landscape +- GitHub Copilot: 60% (strong IDE integration) +- Cursor: 15% (rapid growth, full IDE) +- Tabnine: 10% (enterprise, on-premise) +- Cody: 5% (codebase-aware) +- CodeWhisperer: 8% (AWS integration) +Source: AI Tools Competitive Analysis + +### Technology Trends +Key differentiators: context awareness, customization, multi-modal interfaces, code verification. +Source: Developer Tools Landscape + +## Next Steps +1. Monitor Cursor growth and feature releases +2. Evaluate Cody's codebase-aware capabilities +3. Document enterprise security/compliance requirements +4. Track pricing trends +``` + +## Key Takeaways +- Found relevant pages across multiple teamspaces +- Synthesized competitive, market, and technical perspectives +- Used proper citations linking to source pages +- Created actionable recommendations diff --git a/plugins/notion/skills/notion-research-documentation/examples/technical-investigation.md b/plugins/notion/skills/notion-research-documentation/examples/technical-investigation.md new file mode 100644 index 0000000..f74a471 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/examples/technical-investigation.md @@ -0,0 +1,233 @@ +# Example: Technical Investigation + +## User Request + +> "Research our current caching strategy and create a technical summary" + +## Workflow Execution + +### Step 1: Search for Caching Information + +``` +Notion:search +query: "caching strategy architecture" +query_type: "internal" +teamspace_id: "engineering-teamspace-id" +``` + +**Results Found:** +- "System Architecture Overview" (Engineering) +- "Redis Implementation Guide" (Backend Docs) +- "Performance Optimization - Q3 2024" (Engineering) +- "API Caching Decision Record" (Architecture) + +### Step 2: Fetch Relevant Pages + +``` +Notion:fetch +id: "system-architecture-page-url" +``` + +Extracted: Current caching architecture uses Redis for API responses, Memcached for session storage. + +``` +Notion:fetch +id: "redis-implementation-guide-url" +``` + +Extracted: Implementation details, TTL settings, invalidation strategies. + +``` +Notion:fetch +id: "api-caching-decision-record-url" +``` + +Extracted: Why Redis was chosen over alternatives, trade-offs considered. + +### Step 3: Synthesize Findings + +**Key findings identified:** +- Two-tier caching: Redis (API responses) + Memcached (sessions) +- TTL strategy: 5 minutes for dynamic data, 1 hour for static +- Invalidation: Event-driven for critical updates +- Performance impact: 75% reduction in database load +- Known issues: Cache stampede on popular endpoints + +### Step 4: Create Technical Summary + +``` +Notion:notion-create-pages + +parent: { page_id: "engineering-docs-parent-id" } + +pages: [{ + properties: { + "title": "Technical Summary: Caching Strategy - Oct 2025" + }, + content: "[Structured technical summary using template]" +}] +``` + +## Output Document + +```markdown +# Technical Summary: Caching Strategy - Oct 2025 + +## Executive Summary + +Our caching infrastructure uses a two-tier approach with Redis for API response caching and Memcached for session management. This strategy has reduced database load by 75% and improved API response times from 200ms to 50ms average. + +## Architecture Overview + +### Layer 1: API Response Caching (Redis) +**Technology**: Redis 7.0 cluster (3 nodes) +**Purpose**: Cache GET endpoint responses +**TTL Strategy**: +- Dynamic content: 5 minutes +- Static content: 1 hour +- User-specific: 15 minutes + +**Source**: System Architecture Overview + +### Layer 2: Session Storage (Memcached) +**Technology**: Memcached 1.6 +**Purpose**: User session data, temporary state +**TTL**: 24 hours (session lifetime) + +## Implementation Details + +### Cache Key Format +``` +api:v1:{endpoint}:{params_hash} +session:{user_id}:{session_id} +``` + +### Invalidation Strategy +- **Event-driven**: Critical data changes trigger immediate invalidation +- **Time-based**: TTL expiration for non-critical data +- **Manual**: Admin tools for emergency cache clear + +**Source**: Redis Implementation Guide + +## Decision Rationale + +### Why Redis for API Caching? + +**Pros**: +- Advanced data structures (sorted sets, hashes) +- Built-in TTL with automatic eviction +- Pub/sub for cache invalidation events +- Persistence options for durability + +**Cons**: +- Higher memory usage than Memcached +- More complex cluster management + +**Decision**: Chosen for flexibility and rich feature set needed for API caching. + +**Source**: API Caching Decision Record + +### Why Memcached for Sessions? + +**Pros**: +- Simpler, lighter weight +- Excellent for key-value storage +- Lower memory footprint + +**Cons**: +- No persistence +- Limited data structures + +**Decision**: Perfect fit for ephemeral session data where simplicity is valued. + +## Performance Impact + +| Metric | Before Caching | After Caching | Improvement | +|--------|----------------|---------------|-------------| +| Avg Response Time | 200ms | 50ms | 75% faster | +| Database Load | 100% | 25% | 75% reduction | +| Cache Hit Rate | - | 85% | - | +| Peak RPS Handled | 1,000 | 4,000 | 4x increase | + +**Source**: Performance Optimization - Q3 2024 + +## Known Issues & Limitations + +### Cache Stampede +**Issue**: When popular cache entries expire, multiple requests hit database simultaneously + +**Mitigation**: Implemented probabilistic early expiration and request coalescing + +**Status**: Reduced by 90% but not eliminated + +### Stale Data Risk +**Issue**: Cached data can be up to TTL duration stale + +**Mitigation**: Event-driven invalidation for critical data paths + +**Status**: Acceptable trade-off for performance gains + +## Monitoring & Observability + +**Metrics tracked**: +- Cache hit/miss rates per endpoint +- Memory usage and eviction rates +- Response time distributions +- Invalidation event frequency + +**Tools**: DataDog dashboards, CloudWatch alarms + +## Future Considerations + +1. **Edge Caching**: Evaluate CDN integration for static assets +2. **Cache Warming**: Pre-populate cache for predictable traffic spikes +3. **Adaptive TTLs**: Adjust TTLs based on data change frequency +4. **Regional Caching**: Multi-region cache replication for global performance + +## Related Documentation + +- System Architecture Overview +- Redis Implementation Guide +- API Caching Decision Record +- Performance Optimization - Q3 2024 + +## Appendix: Configuration Examples + +### Redis Configuration +```yaml +maxmemory: 8gb +maxmemory-policy: allkeys-lru +tcp-keepalive: 60 +``` + +### Common Cache Operations +```python +# Set with TTL +cache.set(key, value, ttl=300) + +# Get with fallback +value = cache.get(key) or fetch_from_db(key) + +# Invalidate pattern +cache.delete_pattern("api:v1:users:*") +``` +``` + +## Key Success Factors + +1. **Multiple source integration**: Combined architecture docs, implementation guides, and decision records +2. **Technical depth**: Included configuration, code examples, metrics +3. **Decision context**: Explained why choices were made, not just what +4. **Practical focus**: Real performance numbers and known issues +5. **Future-looking**: Noted areas for improvement +6. **Well-cited**: Every major point links back to source material + +## Workflow Pattern Demonstrated + +This example shows the complete research workflow: +- **Scoped search** (teamspace filter for engineering) +- **Multi-page synthesis** (4 different sources) +- **Technical template** (architecture-focused format) +- **Proper placement** (under engineering docs) +- **Comprehensive citations** (links to all sources) + diff --git a/plugins/notion/skills/notion-research-documentation/examples/trip-planning.md b/plugins/notion/skills/notion-research-documentation/examples/trip-planning.md new file mode 100644 index 0000000..7754f16 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/examples/trip-planning.md @@ -0,0 +1,128 @@ +# Example: Group Trip Research & Planning + +**User Request**: "Research and plan our friends' trip to Japan in March - we're 6 people looking for 10 days" + +## Workflow + +### 1. Search Existing Notes +``` +Notion:search +query: "Japan travel" +``` +Found: Japan Travel Guide (from friend), Tokyo Restaurants, Kyoto Temple Guide + +### 2. Fetch & Extract Tips +``` +Notion:fetch (3x) +``` +**Key info from previous travelers:** +- Best time: March-April (cherry blossoms) +- Must-see: Tokyo, Kyoto, Osaka +- Budget: $200-300/day (mid-range) +- Book accommodations 3 months ahead +- Get JR Pass before arrival +- Top restaurants: Sushi Dai, Ichiran Ramen, Tsunahachi Tempura + +### 3. Research & Synthesize +Combined previous traveler insights with: +- Flight options and prices +- Accommodation types (hotels/ryokans/Airbnb) +- Transportation (JR Pass essential) +- 10-day itinerary structure +- Budget breakdown + +### 4. Create Comprehensive Plan +``` +Notion:notion-create-pages +parent: { page_id: "travel-plans-parent-id" } +pages: [{ + properties: { + title: "Japan Trip 2026 - March 15-25 (10 Days)" + }, + content: "[Full trip plan with itinerary, budget, tips...]" +}] +``` + +## Output Sample + +```markdown +# Japan Trip 2026 - March 15-25 (10 Days) + +## Trip Overview +**Dates**: March 15-25, 2026 (Cherry Blossom Season 🌸) +**Group**: 6 people | **Budget**: $3,000-4,000/person + +## Itinerary Summary + +**Days 1-3: Tokyo** +- Arrive, explore Shibuya, Harajuku, Shinjuku +- Visit Tsukiji Market, Imperial Palace, Sensoji Temple +- Experience Tokyo nightlife, teamLab Borderless + +**Days 4-5: Hakone** +- Day trip from Tokyo +- Hot springs, Mt. Fuji views, Lake Ashi cruise + +**Days 6-8: Kyoto** +- Bullet train from Tokyo +- Fushimi Inari, Kinkaku-ji, Arashiyama Bamboo Grove +- Geisha district (Gion), traditional tea ceremony + +**Days 9-10: Osaka + Depart** +- Osaka Castle, Dotonbori food district +- Nara day trip (deer park, Todai-ji Temple) +- Return flight from Osaka (KIX) + +## Budget Breakdown (per person) +- **Flights**: $900-1,200 (round-trip) +- **Accommodations**: $700-1,000 (9 nights) +- **JR Pass**: $280 (7-day pass) +- **Food**: $500-700 ($50-70/day) +- **Activities**: $300-400 +- **Local transport**: $100 +**Total**: $2,780-3,680 + +## Key Bookings +- **Flights**: Book 3-4 months ahead +- **Hotels**: Book now (cherry blossom season) + - Tokyo: Shinjuku area (2 nights) + Asakusa (1 night) + - Kyoto: Gion area (3 nights) + - Osaka: Namba area (2 nights) +- **JR Pass**: Order 2-3 weeks before travel +- **Restaurants**: Reserve 1 week ahead (Sushi Dai, high-end spots) + +## Essential Tips +Source: Japan Travel Guide +- Get pocket WiFi or eSIM on arrival +- Download: Google Translate, Hyperdia (train routes), Tabelog (restaurants) +- Cash-heavy country - withdraw at 7-Eleven ATMs +- Shoes off in temples, ryokans, some restaurants +- Trains extremely punctual - don't be late +- Learn basic phrases: arigatou, sumimasen, itadakimasu + +## Packing List +- Comfortable walking shoes (10k+ steps/day) +- Light jacket (March 55-65°F) +- Backpack for day trips +- Cash pouch +- Portable charger + +## Next Steps +- [ ] Book flights (target: <$1,100/person) +- [ ] Order JR Passes +- [ ] Book hotels (Tokyo → Kyoto → Osaka) +- [ ] Create shared expense tracker +- [ ] Schedule group planning call + +## Sources +- Japan Travel Guide (Sarah's 2024 trip) +- Tokyo Restaurant Recommendations +- Kyoto Temple Guide +``` + +## Key Takeaways +- Leveraged previous traveler notes from Notion +- Combined personal insights with research +- Created actionable itinerary with budget breakdown +- Included practical tips from experienced travelers +- Set clear next steps for group coordination diff --git a/plugins/notion/skills/notion-research-documentation/reference/advanced-search.md b/plugins/notion/skills/notion-research-documentation/reference/advanced-search.md new file mode 100644 index 0000000..a555b1a --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/advanced-search.md @@ -0,0 +1,212 @@ +# Advanced Search Techniques + +## Search Filtering + +### By Date Range + +Use `created_date_range` to find recent content: + +``` +filters: { + created_date_range: { + start_date: "2024-01-01", + end_date: "2025-01-01" + } +} +``` + +**When to use**: +- Finding recent updates on a topic +- Focusing on current information +- Excluding outdated content + +### By Creator + +Use `created_by_user_ids` to find content from specific people: + +``` +filters: { + created_by_user_ids: ["user-id-1", "user-id-2"] +} +``` + +**When to use**: +- Research from subject matter experts +- Team-specific information +- Attribution tracking + +### Combined Filters + +Stack filters for precision: + +``` +filters: { + created_date_range: { + start_date: "2024-10-01" + }, + created_by_user_ids: ["expert-user-id"] +} +``` + +## Scoped Searches + +### Teamspace Scoping + +Restrict search to specific teamspace: + +``` +teamspace_id: "teamspace-uuid" +``` + +**When to use**: +- Project-specific research +- Department-focused information +- Reducing noise from irrelevant results + +### Page Scoping + +Search within a specific page and its subpages: + +``` +page_url: "https://notion.so/workspace/Page-Title-uuid" +``` + +**When to use**: +- Research within a project hierarchy +- Documentation updates +- Focused investigation + +### Database Scoping + +Search within a database's content: + +``` +data_source_url: "collection://data-source-uuid" +``` + +**When to use**: +- Task/project database research +- Structured data investigation +- Finding specific entries + +## Search Strategies + +### Broad to Narrow + +1. Start with general search term +2. Review results for relevant teamspaces/pages +3. Re-search with scope filters +4. Fetch detailed content from top results + +**Example**: +``` +Search 1: query="API integration" → 50 results across workspace +Search 2: query="API integration", teamspace_id="engineering" → 12 results +Fetch: Top 3-5 most relevant pages +``` + +### Multi-Query Approach + +Run parallel searches with related terms: + +``` +Query 1: "API integration" +Query 2: "API authentication" +Query 3: "API documentation" +``` + +Combine results to build comprehensive picture. + +### Temporal Research + +Search across time periods to track evolution: + +``` +Search 1: created_date_range 2023 → Historical context +Search 2: created_date_range 2024 → Recent developments +Search 3: created_date_range 2025 → Current state +``` + +## Result Processing + +### Identifying Relevant Results + +Look for: +- **High semantic match**: Result summary closely matches query intent +- **Recent updates**: Last-edited date is recent +- **Authoritative sources**: Created by known experts or in official locations +- **Comprehensive content**: Result summary suggests detailed information + +### Prioritizing Fetches + +Fetch pages in order of relevance: + +1. **Primary sources**: Direct documentation, official pages +2. **Recent updates**: Newly edited content +3. **Related context**: Supporting information +4. **Historical reference**: Background and context + +Don't fetch everything - be selective based on research needs. + +### Handling Too Many Results + +If search returns 20+ results: + +1. **Add filters**: Narrow by date, creator, or teamspace +2. **Refine query**: Use more specific terms +3. **Use page scoping**: Search within relevant parent page +4. **Sample strategically**: Fetch diverse results (recent, popular, authoritative) + +### Handling Too Few Results + +If search returns < 3 results: + +1. **Broaden query**: Use more general terms +2. **Remove filters**: Search full workspace +3. **Try synonyms**: Alternative terminology +4. **Search in related areas**: Adjacent teamspaces or pages + +## Search Quality + +### Effective Search Queries + +**Good queries** (specific, semantic): +- "Q4 product roadmap" +- "authentication implementation guide" +- "customer feedback themes" + +**Weak queries** (too vague): +- "roadmap" +- "guide" +- "feedback" + +**Over-specific queries** (too narrow): +- "Q4 2024 product roadmap for mobile app version 3.2 feature X" + +### User Context + +Always use available user context: +- Query should match their terminology +- Scope to their relevant teamspaces +- Consider their role/department +- Reference their recent pages + +## Connected Sources + +### Notion Integrations + +Search extends beyond Notion pages to: +- Slack messages (if connected) +- Google Drive documents (if connected) +- GitHub issues/PRs (if connected) +- Jira tickets (if connected) + +Be aware results may come from these sources. + +### Source Attribution + +When citing results from connected sources: +- Note the source type in documentation +- Use appropriate mention format +- Verify user has access to the source system + diff --git a/plugins/notion/skills/notion-research-documentation/reference/citations.md b/plugins/notion/skills/notion-research-documentation/reference/citations.md new file mode 100644 index 0000000..d3b2cff --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/citations.md @@ -0,0 +1,190 @@ +# Citation Styles + +## Basic Page Citation + +Always cite sources using Notion page mentions: + +```markdown +Page Title +``` + +The URL must be provided. The title is optional but improves readability: + +```markdown + +``` + +## Inline Citations + +Cite immediately after referenced information: + +```markdown +The Q4 revenue increased by 23% quarter-over-quarter (Q4 Financial Report). +``` + +## Multiple Sources + +When information comes from multiple sources: + +```markdown +Customer satisfaction has improved across all metrics (Q3 Survey Results, Support Analysis). +``` + +## Section-Level Citations + +For longer sections derived from one source: + +```markdown +### Engineering Priorities + +According to the Engineering Roadmap 2025: + +- Focus on API scalability +- Improve developer experience +- Migrate to microservices architecture +``` + +## Sources Section + +Always include a "Sources" section at document end: + +```markdown +## Sources + +- Strategic Plan 2025 +- Market Analysis Report +- Competitor Research: Q3 +- Customer Interview Notes +``` + +Group by category for long lists: + +```markdown +## Sources + +### Primary Sources +- Official Roadmap +- Strategy Document + +### Supporting Research +- Market Trends +- Customer Feedback + +### Background Context +- Historical Analysis +``` + +## Quoting Content + +When quoting directly from source: + +```markdown +The product team noted: "We need to prioritize mobile experience improvements" (Product Meeting Notes). +``` + +For block quotes: + +```markdown +> We need to prioritize mobile experience improvements to meet our Q4 goals. This includes performance optimization and UI refresh. +> +> — Product Meeting Notes - Oct 2025 +``` + +## Data Citations + +When presenting data, cite the source: + +```markdown +| Metric | Q3 | Q4 | Change | +|--------|----|----|--------| +| Revenue | $2.3M | $2.8M | +21.7% | +| Users | 12.4K | 15.1K | +21.8% | + +Source: Financial Dashboard +``` + +## Database Citations + +When referencing database content: + +```markdown +Based on analysis of the Projects Database, 67% of projects are on track. +``` + +## User Citations + +When attributing information to specific people: + +```markdown +Sarah Chen noted in Architecture Review that the microservices migration is ahead of schedule. +``` + +## Citation Frequency + +**Over-citing** (every sentence): +```markdown +The revenue increased (Report). +Costs decreased (Report). +Margin improved (Report). +``` + +**Under-citing** (no attribution): +```markdown +The revenue increased, costs decreased, and margin improved. +``` + +**Right balance** (grouped citation): +```markdown +The revenue increased, costs decreased, and margin improved (Q4 Financial Report). +``` + +## Outdated Information + +Note when source information might be outdated: + +```markdown +The original API design (API Spec v1, last updated January 2024) has been superseded by the new architecture in API Spec v2. +``` + +## Cross-References + +Link to related research documents: + +```markdown +## Related Research + +This research builds on previous findings: +- Market Analysis - Q2 2025 +- Competitor Landscape Review + +For implementation details, see: +- Technical Implementation Guide +``` + +## Citation Validation + +Before finalizing research: + +✓ Every key claim has a source citation +✓ All page mentions have valid URLs +✓ Sources section includes all cited pages +✓ Outdated sources are noted as such +✓ Direct quotes are clearly marked +✓ Data sources are attributed + +## Citation Style Consistency + +Choose one citation style and use throughout: + +**Inline style** (lightweight): +```markdown +Revenue grew 23% (Financial Report). Customer count increased 18% (Metrics Dashboard). +``` + +**Formal style** (full mentions): +```markdown +Revenue grew 23% (Q4 Financial Report). Customer count increased 18% (Metrics Dashboard). +``` + +**Recommend formal style** for most research documentation as it provides clickable navigation. + diff --git a/plugins/notion/skills/notion-research-documentation/reference/comparison-format.md b/plugins/notion/skills/notion-research-documentation/reference/comparison-format.md new file mode 100644 index 0000000..cfb31b7 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/comparison-format.md @@ -0,0 +1,37 @@ +# Comparison Format + +**When to use**: +- Evaluating multiple options +- Tool/vendor selection +- Approach comparison +- Decision support + +## Characteristics + +**Length**: 800-1200 words typically + +**Structure**: +- Overview of what's being compared +- Comparison matrix table +- Detailed analysis per option (pros/cons) +- Clear recommendation with rationale +- Sources + +## Template + +See [comparison-template.md](comparison-template.md) for the full template. + +## Best For + +- Decision support with multiple options +- Tool or vendor selection +- Comparing different technical approaches +- Evaluating trade-offs between alternatives + +## Example Use Cases + +- "Compare the three database options discussed in our tech docs" +- "What are the pros and cons of each deployment approach?" +- "Compare the vendor proposals" +- "Evaluate the different authentication methods we've documented" + diff --git a/plugins/notion/skills/notion-research-documentation/reference/comparison-template.md b/plugins/notion/skills/notion-research-documentation/reference/comparison-template.md new file mode 100644 index 0000000..ce3c138 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/comparison-template.md @@ -0,0 +1,44 @@ +# Comparison Template + +Use when researching multiple options or alternatives. See [comparison-format.md](comparison-format.md) for when to use this format. + +```markdown +# [Topic] Comparison + +## Overview +[Brief introduction to what's being compared and why] + +## Comparison Matrix + +| Criteria | Option A | Option B | Option C | +|----------|----------|----------|----------| +| [Criterion 1] | [Rating/Details] | [Rating/Details] | [Rating/Details] | +| [Criterion 2] | [Rating/Details] | [Rating/Details] | [Rating/Details] | + +## Detailed Analysis + +### Option A +**Pros**: +- [Advantage] +- [Advantage] + +**Cons**: +- [Disadvantage] +- [Disadvantage] + +**Best for**: [Use case] + +**Source**: Source Page + +[Repeat for each option] + +## Recommendation + +**Selected option**: [Choice] + +**Rationale**: [Why this option is best given the context] + +## Sources +[List all consulted pages] +``` + diff --git a/plugins/notion/skills/notion-research-documentation/reference/comprehensive-report-format.md b/plugins/notion/skills/notion-research-documentation/reference/comprehensive-report-format.md new file mode 100644 index 0000000..1422b2b --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/comprehensive-report-format.md @@ -0,0 +1,41 @@ +# Comprehensive Report Format + +**When to use**: +- Formal documentation requirements +- Strategic decision support +- Complex topics requiring extensive analysis +- Multiple stakeholders need alignment + +## Characteristics + +**Length**: 1500+ words + +**Structure**: +- Executive summary +- Background & context +- Methodology +- Detailed findings with subsections +- Data & evidence section +- Implications (short and long-term) +- Prioritized recommendations +- Appendix + +## Template + +See [comprehensive-report-template.md](comprehensive-report-template.md) for the full template. + +## Best For + +- Deep analysis and strategic decisions +- Formal documentation requirements +- Complex topics with multiple facets +- When stakeholders need extensive context +- Board presentations or executive briefings + +## Example Use Cases + +- "Create a comprehensive analysis of our market position" +- "Document the full technical investigation of the database migration" +- "Prepare an in-depth report on vendor options for executive review" +- "Analyze the pros and cons of different architectural approaches" + diff --git a/plugins/notion/skills/notion-research-documentation/reference/comprehensive-report-template.md b/plugins/notion/skills/notion-research-documentation/reference/comprehensive-report-template.md new file mode 100644 index 0000000..a49620f --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/comprehensive-report-template.md @@ -0,0 +1,64 @@ +# Comprehensive Report Template + +Use for in-depth research requiring extensive analysis. See [comprehensive-report-format.md](comprehensive-report-format.md) for when to use this format. + +```markdown +# [Report Title] + +## Executive Summary +[One paragraph summarizing the entire report] + +## Background & Context +[Why this research was conducted, what questions it addresses] + +## Methodology +- Sources consulted: [number] Notion pages across [teamspaces] +- Time period: [if relevant] +- Scope: [what was included/excluded] + +## Key Findings + +### [Major Theme 1] +**Summary**: [One sentence] + +**Details**: +- [Supporting point with evidence] +- [Supporting point with evidence] +- [Supporting point with evidence] + +**Sources**: [Page mentions] + +### [Major Theme 2] +[Repeat structure] + +## Data & Evidence + +[Tables, quotes, specific data points] + +## Implications + +### Short-term +[Immediate implications] + +### Long-term +[Strategic implications] + +## Recommendations + +### Priority 1: [High priority action] +- **What**: [Specific action] +- **Why**: [Rationale] +- **How**: [Implementation approach] + +### Priority 2: [Medium priority action] +[Repeat structure] + +## Appendix + +### Additional Resources +- [Related pages] + +### Open Questions +- [Unanswered questions for future research] +``` + diff --git a/plugins/notion/skills/notion-research-documentation/reference/format-selection-guide.md b/plugins/notion/skills/notion-research-documentation/reference/format-selection-guide.md new file mode 100644 index 0000000..87facda --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/format-selection-guide.md @@ -0,0 +1,95 @@ +# Format Selection Guide + +Choose the right output format for your research needs. + +## Decision Tree + +``` +Is this comparing multiple options? + ├─ YES → Use Comparison Format + └─ NO ↓ + +Is this time-sensitive or simple? + ├─ YES → Use Quick Brief + └─ NO ↓ + +Does this require formal/extensive documentation? + ├─ YES → Use Comprehensive Report + └─ NO → Use Research Summary (default) +``` + +## Format Overview + +| Format | Length | When to Use | Template | +|--------|--------|-------------|----------| +| [Research Summary](research-summary-format.md) | 500-1000 words | Most research requests (default) | [Template](research-summary-template.md) | +| [Comprehensive Report](comprehensive-report-format.md) | 1500+ words | Formal docs, strategic decisions | [Template](comprehensive-report-template.md) | +| [Quick Brief](quick-brief-format.md) | 200-400 words | Time-sensitive, simple topics | [Template](quick-brief-template.md) | +| [Comparison](comparison-format.md) | 800-1200 words | Evaluating options | [Template](comparison-template.md) | + +## Formatting Guidelines + +### Headings +- Use `#` for title +- Use `##` for major sections +- Use `###` for subsections +- Keep heading hierarchy consistent + +### Lists +- Use `-` for bullet points +- Use `1.` for numbered lists +- Keep list items parallel in structure + +### Emphasis +- Use `**bold**` for key terms and section labels +- Use `*italic*` for emphasis +- Use sparingly for maximum impact + +### Citations +- Always use `Page Title` for source pages +- Include citation immediately after referenced information +- Group all sources in a "Sources" section at the end + +### Tables +- Use for structured data comparison +- Keep columns to 3-5 for readability +- Include header row +- Align content appropriately + +### Code Blocks +Use when including: +- Technical specifications +- Configuration examples +- Command examples + +``` +Example code or configuration here +``` + +## Content Guidelines + +### Executive Summaries +- Lead with the most important finding +- Include 1-2 key implications +- Make it standalone (reader gets value without reading further) +- Target 2-3 sentences for summaries, 1 paragraph for reports + +### Key Findings +- Start with a clear headline +- Support with specific evidence +- Include relevant data points or quotes +- Cite source immediately +- Focus on actionable insights + +### Recommendations +- Make them specific and actionable +- Explain the "why" behind each recommendation +- Prioritize clearly (Priority 1, 2, 3 or High/Medium/Low) +- Include implementation hints when relevant + +### Source Citations +- Link to original pages using mentions +- Note if information is outdated (check last-edited dates) +- Credit specific sections when quoting +- Group related sources together + diff --git a/plugins/notion/skills/notion-research-documentation/reference/quick-brief-format.md b/plugins/notion/skills/notion-research-documentation/reference/quick-brief-format.md new file mode 100644 index 0000000..41ef5f2 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/quick-brief-format.md @@ -0,0 +1,37 @@ +# Quick Brief Format + +**When to use**: +- Time-sensitive requests +- Simple topics +- Status updates +- Quick reference needs + +## Characteristics + +**Length**: 200-400 words + +**Structure**: +- 3-4 sentence summary +- 3-5 bullet key points +- Short action items list +- Brief source list + +## Template + +See [quick-brief-template.md](quick-brief-template.md) for the full template. + +## Best For + +- Fast turnaround requests +- Simple, straightforward topics +- Quick status updates +- When time is more important than depth +- Initial exploration before deeper research + +## Example Use Cases + +- "Quick summary of what's in our API docs" +- "Fast brief on the meeting notes from yesterday" +- "What are the key points from that spec?" +- "Give me a quick overview of the project status" + diff --git a/plugins/notion/skills/notion-research-documentation/reference/quick-brief-template.md b/plugins/notion/skills/notion-research-documentation/reference/quick-brief-template.md new file mode 100644 index 0000000..2b3f723 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/quick-brief-template.md @@ -0,0 +1,25 @@ +# Quick Brief Template + +Use for fast turnaround requests or simple topics. See [quick-brief-format.md](quick-brief-format.md) for when to use this format. + +```markdown +# [Topic] - Quick Brief + +**Date**: [Current date] + +## Summary +[3-4 sentences covering the essentials] + +## Key Points +- **Point 1**: [Details] +- **Point 2**: [Details] +- **Point 3**: [Details] + +## Action Items +1. [Immediate next step] +2. [Follow-up action] + +## Sources +[Brief list of pages consulted] +``` + diff --git a/plugins/notion/skills/notion-research-documentation/reference/research-summary-format.md b/plugins/notion/skills/notion-research-documentation/reference/research-summary-format.md new file mode 100644 index 0000000..ce26657 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/research-summary-format.md @@ -0,0 +1,33 @@ +# Research Summary Format + +**When to use**: General research requests, most common format + +## Characteristics + +**Length**: 500-1000 words typically + +**Structure**: +- Executive summary (2-3 sentences) +- 3-5 key findings with supporting evidence +- Detailed analysis section +- Conclusions and next steps +- Source citations + +## Template + +See [research-summary-template.md](research-summary-template.md) for the full template. + +## Best For + +- Most general-purpose research requests +- Standard documentation needs +- Balanced depth and readability +- When you need comprehensive but accessible information + +## Example Use Cases + +- "Research our authentication options" +- "What does our project documentation say about the API redesign?" +- "Summarize the team's discussion about mobile strategy" +- "Compile information about our deployment process" + diff --git a/plugins/notion/skills/notion-research-documentation/reference/research-summary-template.md b/plugins/notion/skills/notion-research-documentation/reference/research-summary-template.md new file mode 100644 index 0000000..9ebd156 --- /dev/null +++ b/plugins/notion/skills/notion-research-documentation/reference/research-summary-template.md @@ -0,0 +1,49 @@ +# Research Summary Template + +Use this for most research requests. See [research-summary-format.md](research-summary-format.md) for when to use this format. + +```markdown +# [Topic Name] + +## Executive Summary +[2-3 sentence overview of key findings and implications] + +## Key Findings + +### Finding 1: [Clear headline] +[Details and supporting evidence] +- Source: Original Page + +### Finding 2: [Clear headline] +[Details and supporting evidence] +- Source: Original Page + +### Finding 3: [Clear headline] +[Details and supporting evidence] +- Source: Original Page + +## Detailed Analysis + +### [Section 1] +[In-depth discussion of first major theme] + +### [Section 2] +[In-depth discussion of second major theme] + +## Conclusions + +[Summary of implications and insights] + +## Next Steps + +1. [Actionable recommendation] +2. [Actionable recommendation] +3. [Actionable recommendation] + +## Sources + +- Page Title +- Page Title +- Page Title +``` + diff --git a/plugins/notion/skills/notion-spec-to-implementation/LICENSE.txt b/plugins/notion/skills/notion-spec-to-implementation/LICENSE.txt new file mode 100644 index 0000000..0871708 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/LICENSE.txt @@ -0,0 +1,7 @@ +Copyright 2025 Notion Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/notion/skills/notion-spec-to-implementation/SKILL.md b/plugins/notion/skills/notion-spec-to-implementation/SKILL.md new file mode 100644 index 0000000..73918bd --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/SKILL.md @@ -0,0 +1,58 @@ +--- +name: notion-spec-to-implementation +description: Turn Notion specs into implementation plans, tasks, and progress tracking; use when implementing PRDs/feature specs and creating Notion plans + tasks from them. +metadata: + short-description: Turn Notion specs into implementation plans, tasks, and progress tracking +--- + +# Spec to Implementation + +Convert a Notion spec into linked implementation plans, tasks, and ongoing status updates. + +## Quick start +1) Locate the spec with `Notion:search`, then fetch it with `Notion:fetch`. +2) Parse requirements and ambiguities using `reference/spec-parsing.md`. +3) Create a plan page with `Notion:notion-create-pages` (pick a template: quick vs. full). +4) Find the task database, confirm schema, then create tasks with `Notion:notion-create-pages`. +5) Link spec ↔ plan ↔ tasks; keep status current with `Notion:notion-update-page`. + +## Workflow + +### 0) If any MCP call fails because Notion MCP is not connected, pause and set it up: +1. Add the Notion MCP: + - `codex mcp add notion --url https://mcp.notion.com/mcp` +2. Enable remote MCP client: + - Set `[features].rmcp_client = true` in `config.toml` **or** run `codex --enable rmcp_client` +3. Log in with OAuth: + - `codex mcp login notion` + +After successful login, the user will have to restart codex. You should finish your answer and tell them so when they try again they can continue with Step 1. + +### 1) Locate and read the spec +- Search first (`Notion:search`); if multiple hits, ask the user which to use. +- Fetch the page (`Notion:fetch`) and scan for requirements, acceptance criteria, constraints, and priorities. See `reference/spec-parsing.md` for extraction patterns. +- Capture gaps/assumptions in a clarifications block before proceeding. + +### 2) Choose plan depth +- Simple change → use `reference/quick-implementation-plan.md`. +- Multi-phase feature/migration → use `reference/standard-implementation-plan.md`. +- Create the plan via `Notion:notion-create-pages`, include: overview, linked spec, requirements summary, phases, dependencies/risks, and success criteria. Link back to the spec. + +### 3) Create tasks +- Find the task database (`Notion:search` → `Notion:fetch` to confirm the data source and required properties). Patterns in `reference/task-creation.md`. +- Size tasks to 1–2 days. Use `reference/task-creation-template.md` for content (context, objective, acceptance criteria, dependencies, resources). +- Set properties: title/action verb, status, priority, relations to spec + plan, due date/story points/assignee if provided. +- Create pages with `Notion:notion-create-pages` using the database’s `data_source_id`. + +### 4) Link artifacts +- Plan links to spec; tasks link to both plan and spec. +- Optionally update the spec with a short “Implementation” section pointing to the plan and tasks using `Notion:notion-update-page`. + +### 5) Track progress +- Use the cadence in `reference/progress-tracking.md`. +- Post updates with `reference/progress-update-template.md`; close phases with `reference/milestone-summary-template.md`. +- Keep checklists and status fields in plan/tasks in sync; note blockers and decisions. + +## References and examples +- `reference/` — parsing patterns, plan/task templates, progress cadence (e.g., `spec-parsing.md`, `standard-implementation-plan.md`, `task-creation.md`, `progress-tracking.md`). +- `examples/` — end-to-end walkthroughs (e.g., `ui-component.md`, `api-feature.md`, `database-migration.md`). diff --git a/plugins/notion/skills/notion-spec-to-implementation/agents/openai.yaml b/plugins/notion/skills/notion-spec-to-implementation/agents/openai.yaml new file mode 100644 index 0000000..73a5795 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/agents/openai.yaml @@ -0,0 +1,14 @@ +interface: + display_name: "Notion Spec to Implementation" + short_description: "Turn Notion specs into implementation plans, tasks, and progress tracking" + icon_small: "./assets/notion-small.svg" + icon_large: "./assets/notion.png" + default_prompt: "Turn this Notion spec into an implementation plan with milestones, tasks, and dependencies." + +dependencies: + tools: + - type: "mcp" + value: "notion" + description: "Notion MCP server" + transport: "streamable_http" + url: "https://mcp.notion.com/mcp" diff --git a/plugins/notion/skills/notion-spec-to-implementation/assets/notion-small.svg b/plugins/notion/skills/notion-spec-to-implementation/assets/notion-small.svg new file mode 100644 index 0000000..455d4be --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/assets/notion-small.svg @@ -0,0 +1,11 @@ + diff --git a/plugins/notion/skills/notion-spec-to-implementation/assets/notion.png b/plugins/notion/skills/notion-spec-to-implementation/assets/notion.png new file mode 100644 index 0000000..085023c Binary files /dev/null and b/plugins/notion/skills/notion-spec-to-implementation/assets/notion.png differ diff --git a/plugins/notion/skills/notion-spec-to-implementation/evaluations/README.md b/plugins/notion/skills/notion-spec-to-implementation/evaluations/README.md new file mode 100644 index 0000000..fe5399f --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/evaluations/README.md @@ -0,0 +1,120 @@ +# Spec to Implementation Skill Evaluations + +Evaluation scenarios for testing the Spec to Implementation skill across different Codex models. + +## Purpose + +These evaluations ensure the Spec to Implementation skill: +- Finds and parses specification pages accurately +- Breaks down specs into actionable implementation plans +- Creates tasks that Codex can implement with clear acceptance criteria +- Tracks progress and updates implementation status +- Works consistently across Haiku, Sonnet, and Opus + +## Evaluation Files + +### basic-spec-implementation.json +Tests basic workflow of turning a spec into an implementation plan. + +**Scenario**: Implement user authentication feature from spec +**Key Behaviors**: +- Searches for and finds the authentication spec page +- Fetches spec and extracts requirements +- Parses requirements into phases (setup, core features, polish) +- Creates implementation plan page linked to original spec +- Breaks down into clear phases with deliverables +- Includes timeline and dependencies + +### spec-to-tasks.json +Tests creating concrete tasks from a specification in a task database. + +**Scenario**: Create tasks from API redesign spec +**Key Behaviors**: +- Finds spec page in Notion +- Extracts specific requirements and acceptance criteria +- Searches for or creates task database +- Fetches task database schema +- Creates multiple tasks with proper properties (Status, Priority, Sprint, etc.) +- Each task has clear title, description, and acceptance criteria +- Tasks have dependencies where appropriate +- Links all tasks back to original spec + +## Running Evaluations + +1. Enable the `spec-to-implementation` skill +2. Submit the query from the evaluation file +3. Verify the skill finds the spec page via search +4. Check that requirements are accurately parsed +5. Confirm implementation plan is created with phases +6. Verify tasks have clear, implementable acceptance criteria +7. Check that tasks link back to spec +8. Test with Haiku, Sonnet, and Opus + +## Expected Skill Behaviors + +Spec to Implementation evaluations should verify: + +### Spec Discovery & Parsing +- Searches Notion for specification pages +- Fetches complete spec content +- Extracts all requirements accurately +- Identifies technical dependencies +- Understands acceptance criteria +- Notes any ambiguities or missing details + +### Implementation Planning +- Creates implementation plan page +- Breaks work into logical phases: + - Phase 1: Foundation/Setup + - Phase 2: Core Implementation + - Phase 3: Testing & Polish +- Includes timeline estimates +- Identifies dependencies between phases +- Links back to original spec + +### Task Creation +- Finds or identifies task database +- Fetches database schema for property names +- Creates tasks with correct properties +- Each task has: + - Clear, specific title + - Context and description + - Acceptance criteria (checklist format) + - Appropriate priority and status + - Link to spec page +- Tasks are right-sized (not too big, not too small) +- Dependencies between tasks are noted + +### Progress Tracking +- Implementation plan includes progress markers +- Tasks can be updated as work progresses +- Status updates link to completed work +- Blockers or changes are noted + +## Creating New Evaluations + +When adding Spec to Implementation evaluations: + +1. **Test different spec types** - Features, migrations, refactors, API changes, UI components +2. **Vary complexity** - Simple 1-phase vs. complex multi-phase implementations +3. **Test task granularity** - Does it create appropriately-sized tasks? +4. **Include edge cases** - Vague specs, conflicting requirements, missing details +5. **Test database integration** - Creating tasks in existing task databases with various schemas +6. **Progress tracking** - Updating implementation plans as tasks complete + +## Example Success Criteria + +**Good** (specific, testable): +- "Searches Notion for spec page using feature name" +- "Creates implementation plan with 3 phases: Setup → Core → Polish" +- "Creates 5-8 tasks in task database with properties: Task (title), Status, Priority, Sprint" +- "Each task has acceptance criteria in checklist format (- [ ] ...)" +- "Tasks link back to spec using mention-page tag" +- "Task titles are specific and actionable (e.g., 'Create login API endpoint' not 'Authentication')" + +**Bad** (vague, untestable): +- "Creates good implementation plan" +- "Tasks are well-structured" +- "Breaks down spec appropriately" +- "Links to spec" + diff --git a/plugins/notion/skills/notion-spec-to-implementation/evaluations/basic-spec-implementation.json b/plugins/notion/skills/notion-spec-to-implementation/evaluations/basic-spec-implementation.json new file mode 100644 index 0000000..7da2ea4 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/evaluations/basic-spec-implementation.json @@ -0,0 +1,32 @@ +{ + "name": "Create Implementation Plan from Spec", + "skills": ["spec-to-implementation"], + "query": "Create an implementation plan for the User Authentication spec page", + "expected_behavior": [ + "Step 1: Uses Notion:search to find 'User Authentication spec' with keywords like 'User Authentication' or 'auth spec'", + "Step 2: If not found or ambiguous, asks user for spec page URL/ID", + "Step 3: Fetches spec page using Notion:fetch with URL/ID from search results", + "Step 4: Parses spec using patterns from reference/spec-parsing.md to extract requirements, acceptance criteria, constraints", + "Step 5: Identifies functional requirements (user stories, features, workflows) and non-functional requirements (performance, security)", + "Step 6: Creates implementation plan following structure from reference/templates.md", + "Step 7: Includes sections: Overview, Linked Spec, Requirements Summary, Technical Approach, Implementation Phases", + "Step 8: Breaks work into logical phases with Goal, Tasks checklist, Estimated effort per phase", + "Step 9: Identifies dependencies and risks from spec content", + "Step 10: Links plan back to original spec page using ", + "Step 11: Creates plan page using Notion:notion-create-pages with appropriate title (e.g., 'Implementation Plan: User Authentication')", + "Step 12: Places plan appropriately (asks user or suggests under project/spec parent)" + ], + "success_criteria": [ + "Spec is found using Notion:search before attempting to fetch (or user is asked for URL if not found)", + "Spec is fetched using Notion:fetch with correct URL/ID from search results", + "Plan includes clear overview and spec link with mention-page tag", + "Requirements are extracted from actual spec content (not generic) using spec-parsing patterns", + "Work is broken into multiple phases (typically 3-5) following template structure", + "Each phase has Goal, Tasks (as checkboxes), and Estimated effort", + "Dependencies and risks sections are included with specific details from spec", + "Plan follows Implementation Plan structure from reference/templates.md", + "Success criteria or acceptance criteria from spec are referenced in plan", + "Uses correct tool names and sequence: Notion:search → Notion:fetch → Notion:notion-create-pages" + ] +} + diff --git a/plugins/notion/skills/notion-spec-to-implementation/evaluations/spec-to-tasks.json b/plugins/notion/skills/notion-spec-to-implementation/evaluations/spec-to-tasks.json new file mode 100644 index 0000000..ee01e32 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/evaluations/spec-to-tasks.json @@ -0,0 +1,35 @@ +{ + "name": "Create Tasks from Specification", + "skills": ["spec-to-implementation", "task-manager"], + "query": "Read the Payment Integration spec and create implementation tasks in our Tasks database", + "expected_behavior": [ + "Step 1: Uses Notion:search to find Payment Integration spec or asks for URL", + "Step 2: Fetches spec page using Notion:fetch to read full content", + "Step 3: Parses spec using reference/spec-parsing.md patterns to identify work items", + "Step 4: Breaks down into appropriately-sized tasks using breakdown patterns from reference/task-creation.md", + "Step 5: Uses Notion:search to find Tasks database location", + "Step 6: Fetches Tasks database using Notion:fetch to get schema, property names, and data sources", + "Step 7: Identifies correct data source from tags in fetch results", + "Step 8: Optionally creates implementation plan page first (recommended per workflow)", + "Step 9: For each task - creates task page using Notion:notion-create-pages with parent: { data_source_id: 'collection://...' }", + "Step 10: Sets task properties from schema: Title, Status (To Do), Priority, Related Tasks (link to spec)", + "Step 11: Includes task description with context, acceptance criteria from spec, dependencies", + "Step 12: Links tasks to spec page using and to each other for dependencies", + "Step 13: Sequences tasks appropriately (setup → implementation → testing per reference/task-creation.md)", + "Step 14: Reports summary: 'Created X tasks for Payment Integration: [task list with links]'" + ], + "success_criteria": [ + "Spec is found using Notion:search before attempting to fetch", + "Task database is found using Notion:search before attempting to fetch schema", + "Database schema is fetched and data source identified from tags", + "Multiple tasks created (at least 3-5) covering spec scope", + "Tasks are right-sized per reference/task-creation.md (1-2 days each)", + "Each task has clear acceptance criteria extracted from spec", + "Tasks are properly sequenced with dependencies using relation properties", + "All tasks link back to original spec using mention-page tags", + "Task properties match exact schema from fetched database", + "Tasks use correct parent (data_source_id: 'collection://...') for database placement", + "Uses correct tool names and sequence: Notion:search (2x) → Notion:fetch (2x) → Notion:notion-create-pages (Nx)" + ] +} + diff --git a/plugins/notion/skills/notion-spec-to-implementation/examples/api-feature.md b/plugins/notion/skills/notion-spec-to-implementation/examples/api-feature.md new file mode 100644 index 0000000..1f4551c --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/examples/api-feature.md @@ -0,0 +1,461 @@ +# Example: API Feature Implementation + +## User Request + +> "Create an implementation plan for the User Profile API spec" + +## Workflow Execution + +### Step 1: Fetch Specification + +``` +Notion:search +query: "User Profile API spec" +query_type: "internal" +``` + +Found: "User Profile API Specification" in Engineering Specs + +``` +Notion:fetch +id: "user-profile-api-spec-page-id" +``` + +### Step 2: Parse Specification + +**Spec Content Extracted:** + +```markdown +# User Profile API Specification + +## Overview +RESTful API for user profile management + +## Requirements + +### Functional +- FR-1: Get user profile by ID +- FR-2: Update user profile (name, bio, avatar) +- FR-3: Upload profile avatar +- FR-4: Get user's public profile (limited fields) +- FR-5: Search users by name + +### Non-Functional +- NFR-1: Response time < 200ms (p95) +- NFR-2: Support 1000 concurrent users +- NFR-3: Avatar upload < 5MB +- NFR-4: GDPR compliant (data portability) + +## API Endpoints + +GET /api/v1/users/:id +PUT /api/v1/users/:id +POST /api/v1/users/:id/avatar +GET /api/v1/users/:id/public +GET /api/v1/users/search + +## Data Model +- id (UUID) +- email (string, unique) +- name (string) +- bio (text, max 500 chars) +- avatar_url (string) +- created_at (timestamp) +- updated_at (timestamp) + +## Security +- Authentication: JWT bearer token +- Authorization: Users can only update own profile +- Rate limiting: 100 req/min per user + +## Acceptance Criteria +- AC-1: All endpoints return proper HTTP status codes +- AC-2: Validation errors return 400 with error details +- AC-3: Unauthorized access returns 401 +- AC-4: Rate limit exceeded returns 429 +- AC-5: Avatar images stored in S3 +``` + +### Step 3: Create Implementation Plan + +``` +Notion:notion-create-pages + +parent: { page_id: "engineering-plans-parent-id" } + +pages: [{ + properties: { + "title": "Implementation Plan: User Profile API" + }, + content: "[Implementation plan]" +}] +``` + +**Implementation Plan Created:** + +```markdown +# Implementation Plan: User Profile API + +## Overview +Build RESTful API for user profile management with CRUD operations, avatar upload, and search functionality. + +## Linked Specification +User Profile API Specification + +## Requirements Summary + +### Functional Requirements +- ✅ Get user profile (authenticated) +- ✅ Update profile fields (name, bio) +- ✅ Upload avatar with image processing +- ✅ Public profile view (limited fields) +- ✅ User search by name + +### Non-Functional Requirements +- **Performance**: < 200ms response time (p95) +- **Scalability**: 1000 concurrent users +- **Storage**: Avatar files < 5MB to S3 +- **Compliance**: GDPR data portability + +### Acceptance Criteria +- All endpoints with proper status codes +- Input validation with error details +- JWT authentication required +- Rate limiting enforced +- Avatars stored in S3 + +## Technical Approach + +### Architecture +- **Framework**: Express.js (Node.js) +- **Database**: PostgreSQL +- **Storage**: AWS S3 for avatars +- **Cache**: Redis for profile data +- **Search**: PostgreSQL full-text search + +### Key Design Decisions +1. **JWT Authentication**: Stateless auth, scales horizontally +2. **S3 for Avatars**: Offload storage, CDN integration ready +3. **Redis Caching**: Reduce DB load for frequently accessed profiles +4. **Rate Limiting**: Token bucket algorithm, per-user limits + +## Implementation Phases + +### Phase 1: Foundation (Days 1-2) +**Goal**: Set up core infrastructure + +**Tasks**: +- [ ] Setup database schema +- [ ] Configure S3 bucket +- [ ] Setup Redis cache +- [ ] Create API scaffolding + +**Deliverables**: Working skeleton with DB, storage, cache ready +**Estimated effort**: 2 days + +### Phase 2: Core Endpoints (Days 3-5) +**Goal**: Implement main CRUD operations + +**Tasks**: +- [ ] Implement GET user profile +- [ ] Implement PUT update profile +- [ ] Add input validation +- [ ] Add JWT authentication middleware +- [ ] Implement rate limiting + +**Deliverables**: Working CRUD operations with auth +**Estimated effort**: 3 days + +### Phase 3: Avatar Upload (Days 6-7) +**Goal**: Avatar management with S3 + +**Tasks**: +- [ ] Implement avatar upload endpoint +- [ ] Add image validation (size, format) +- [ ] Process and resize images +- [ ] Upload to S3 with signed URLs + +**Deliverables**: Avatar upload/update functionality +**Estimated effort**: 2 days + +### Phase 4: Search & Public Profile (Days 8-9) +**Goal**: Complete remaining features + +**Tasks**: +- [ ] Implement user search +- [ ] Implement public profile endpoint +- [ ] Add search indexing +- [ ] Optimize search queries + +**Deliverables**: Search and public profiles working +**Estimated effort**: 2 days + +### Phase 5: Testing & Optimization (Days 10-12) +**Goal**: Production-ready quality + +**Tasks**: +- [ ] Write unit tests +- [ ] Write integration tests +- [ ] Performance testing +- [ ] Security audit +- [ ] API documentation + +**Deliverables**: Tested, documented, production-ready API +**Estimated effort**: 3 days + +## Dependencies + +### External Dependencies +- AWS S3 bucket created ✅ +- Redis instance available ✅ +- PostgreSQL database provisioned ✅ + +### Internal Dependencies +- JWT authentication service (exists) +- User database table (exists) +- Logging infrastructure (exists) + +### Blockers +None currently + +## Risks & Mitigation + +### Risk 1: Image Processing Performance +- **Probability**: Medium +- **Impact**: Medium +- **Mitigation**: Use background job queue for processing, return signed upload URL immediately + +### Risk 2: S3 Upload Failures +- **Probability**: Low +- **Impact**: Medium +- **Mitigation**: Implement retry logic with exponential backoff, fallback to local storage temporarily + +### Risk 3: Rate Limiting Complexity +- **Probability**: Low +- **Impact**: Low +- **Mitigation**: Use proven library (express-rate-limit with Redis store) + +### Risk 4: Search Performance +- **Probability**: Medium +- **Impact**: Medium +- **Mitigation**: Add database indexes, consider Elasticsearch if needed later + +## Timeline + +| Milestone | Target Date | Status | +|-----------|-------------|--------| +| Phase 1 Complete | Oct 16 | ⏳ Planned | +| Phase 2 Complete | Oct 19 | ⏳ Planned | +| Phase 3 Complete | Oct 21 | ⏳ Planned | +| Phase 4 Complete | Oct 23 | ⏳ Planned | +| Phase 5 Complete | Oct 26 | ⏳ Planned | +| **Production Deploy** | **Oct 28** | ⏳ Planned | + +**Total Duration**: 12 working days (~2.5 weeks) + +## Success Criteria + +### Technical Success +- [ ] All 5 endpoints implemented and working +- [ ] Response time < 200ms (p95) verified in load testing +- [ ] Handles 1000 concurrent users +- [ ] All acceptance criteria met +- [ ] Test coverage > 80% +- [ ] Security scan passed +- [ ] API documentation complete + +### Business Success +- [ ] User profile updates functional +- [ ] Avatar uploads working reliably +- [ ] Search returns relevant results in < 500ms +- [ ] Zero critical bugs in first week + +## Resources + +### Documentation +- User Profile API Specification (original spec) +- Authentication Service Docs +- AWS S3 Setup Guide + +### Related Work +- User Authentication API (similar pattern) +- File Upload Service (avatar upload reference) + +### External References +- Express.js best practices +- AWS S3 SDK documentation +- PostgreSQL full-text search guide + +## Progress Tracking + +### Phase Status +- Phase 1: ⏳ Not Started +- Phase 2: ⏳ Not Started +- Phase 3: ⏳ Not Started +- Phase 4: ⏳ Not Started +- Phase 5: ⏳ Not Started + +**Overall Progress**: 0% complete + +### Latest Update +*Implementation plan created on October 14, 2025* +``` + +### Step 4: Find Task Database + +``` +Notion:search +query: "Tasks database" +query_type: "internal" +``` + +Found: "Engineering Tasks" database + +### Step 5: Fetch Task Database Schema + +``` +Notion:fetch +id: "tasks-database-id" +``` + +**Schema retrieved:** +- Data source: `collection://tasks-db-uuid` +- Properties: Name (title), Status (select), Priority (select), Related Tasks (relation), Story Points (number), Tags (multi_select) + +### Step 6: Create Implementation Tasks + +Now create tasks for Phase 1: + +**Task 1: Setup database schema** + +``` +Notion:notion-create-pages + +parent: { data_source_id: "collection://tasks-db-uuid" } + +pages: [{ + properties: { + "Name": "Setup database schema for User Profile API", + "Status": "To Do", + "Priority": "High", + "Related Tasks": ["impl-plan-page-id", "spec-page-id"], + "Story Points": 3, + "Tags": "backend, database, api" + }, + content: "## Context\nImplementation task for User Profile API Specification\n\nPart of Implementation Plan: User Profile API - Phase 1\n\n## Objective\nCreate database schema for user profile storage\n\n## Requirements\nBased on spec data model:\n- id (UUID, primary key)\n- email (string, unique index)\n- name (string, not null)\n- bio (text, max 500 chars)\n- avatar_url (string, nullable)\n- created_at (timestamp)\n- updated_at (timestamp)\n\n## Acceptance Criteria\n- [ ] Migration file created\n- [ ] Schema includes all required fields\n- [ ] Indexes on email (unique) and name (search)\n- [ ] Constraints validated (bio length, email format)\n- [ ] Migration tested on dev database\n- [ ] Rollback migration created\n\n## Technical Approach\n```sql\nCREATE TABLE user_profiles (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n email VARCHAR(255) UNIQUE NOT NULL,\n name VARCHAR(255) NOT NULL,\n bio TEXT CHECK (length(bio) <= 500),\n avatar_url TEXT,\n created_at TIMESTAMP DEFAULT NOW(),\n updated_at TIMESTAMP DEFAULT NOW()\n);\n\nCREATE INDEX idx_user_profiles_email ON user_profiles(email);\nCREATE INDEX idx_user_profiles_name ON user_profiles USING gin(to_tsvector('english', name));\n```\n\n## Dependencies\n- Blocked By: None\n- Blocks: All Phase 2 tasks\n\n## Estimated Effort\n3 story points (half day)\n" +}] +``` + +*[Create similar tasks for all phases - 20 tasks total]* + +### Step 7: Link Plan Back to Spec + +``` +Notion:notion-update-page +page_id: "user-profile-api-spec-page-id" +command: "insert_content_after" +selection_with_ellipsis: "## Acceptance Criteria..." +new_str: " + +--- + +## Implementation + +**Implementation Plan**: Implementation Plan: User Profile API + +**Implementation Tasks**: See plan for full task breakdown (20 tasks across 5 phases) + +**Status**: Planning complete, ready to start implementation +" +``` + +## Summary Provided to User + +```markdown +## Implementation Plan Created ✅ + +### Plan Overview +- **Feature**: User Profile API +- **Duration**: 12 days (~2.5 weeks) +- **Phases**: 5 (Foundation → Core → Avatar → Search → Testing) +- **Tasks**: 20 tasks created +- **Target Launch**: October 28, 2025 + +### Implementation Phases + +**Phase 1: Foundation** (2 days) +- Database schema +- S3 and Redis setup +- API scaffolding + +**Phase 2: Core Endpoints** (3 days) +- GET/PUT user profile +- Authentication & validation +- Rate limiting + +**Phase 3: Avatar Upload** (2 days) +- Image upload & validation +- S3 integration +- Image processing + +**Phase 4: Search & Public Profile** (2 days) +- User search +- Public profile endpoint + +**Phase 5: Testing & Optimization** (3 days) +- Unit & integration tests +- Performance testing +- Documentation + +### Key Deliverables +- 5 REST API endpoints +- Avatar upload to S3 +- User search functionality +- Comprehensive tests +- API documentation + +### Links Created +✅ Plan page: Implementation Plan: User Profile API +✅ Spec updated with plan link +✅ 20 tasks created in task database +✅ All tasks linked to plan and spec + +### Next Steps +1. Review and approve plan +2. Assign tasks to team members +3. Begin Phase 1 (Foundation) +4. Daily standups for progress tracking + +**Ready to start implementation! 🚀** +``` + +## Key Features Demonstrated + +### Spec Parsing +- Extracted requirements (functional & non-functional) +- Identified API endpoints +- Noted data model +- Captured acceptance criteria +- Understood security requirements + +### Implementation Planning +- Broke into logical phases +- Sequenced work appropriately (foundation → features → testing) +- Identified dependencies +- Estimated effort per phase +- Created realistic timeline + +### Task Creation +- Generated 20 specific tasks +- Each task has context, acceptance criteria, technical approach +- Tasks link to both spec and plan +- Proper dependencies noted + +### Bidirectional Linking +- Plan links to spec +- Spec updated to link to plan +- Tasks link to both +- Easy navigation between all artifacts + +Perfect for: Feature implementation, API development, technical projects + diff --git a/plugins/notion/skills/notion-spec-to-implementation/examples/database-migration.md b/plugins/notion/skills/notion-spec-to-implementation/examples/database-migration.md new file mode 100644 index 0000000..d504533 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/examples/database-migration.md @@ -0,0 +1,81 @@ +# Example: Database Migration Implementation + +**User Request**: "Plan and implement the database migration for user preferences schema" + +## Workflow + +### 1. Find & Fetch Spec +``` +Notion:search → Found "User Preferences Schema Migration Spec" +Notion:fetch → Extracted requirements +``` + +**Spec Summary**: Migrate from JSON blob to structured schema for better performance and data integrity. + +### 2. Parse Requirements +- **Current**: JSONB preferences column +- **Target**: Separate `user_preferences` and `notification_preferences` tables +- **Must maintain**: Backward compatibility during migration +- **Performance**: Support 1M+ users with zero downtime + +### 3. Create Implementation Plan +``` +Notion:notion-create-pages +pages: [{ + properties: { + title: "Implementation Plan: User Preferences Migration" + }, + content: "[Full implementation plan with phases]" +}] +``` + +**Plan included**: +- **Phase 1**: Create new tables with indexes +- **Phase 2**: Backfill data from JSONB +- **Phase 3**: Dual-write mode (both old and new) +- **Phase 4**: Switch reads to new schema +- **Phase 5**: Drop old JSONB column + +### 4. Find Task Database & Create Tasks +``` +Notion:search → Found "Engineering Tasks" database +Notion:fetch → Got schema (Task, Status, Priority, Assignee, etc.) + +Notion:notion-create-pages +parent: { data_source_id: "collection://xyz" } +pages: [ + { + properties: { + "Task": "Write migration SQL scripts", + "Status": "To Do", + "Priority": "High", + "Sprint": "Sprint 25" + }, + content: "## Context\nPart of User Preferences Migration...\n\n## Acceptance Criteria\n- [ ] Migration script creates tables\n- [ ] Indexes defined..." + }, + // ... 4 more tasks +] +``` + +**Tasks created**: +1. Write migration SQL scripts +2. Implement backfill job +3. Add dual-write logic to API +4. Update read queries +5. Rollback plan & monitoring + +### 5. Track Progress +Regular updates to implementation plan with status, blockers, and completion notes. + +## Key Outputs + +**Implementation Plan Page** (linked to spec) +**5 Tasks in Database** (with dependencies, acceptance criteria) +**Progress Tracking** (updated as work progresses) + +## Success Factors +- Broke down complex migration into clear phases +- Created tasks with specific acceptance criteria +- Established dependencies (Phase 1 → 2 → 3 → 4 → 5) +- Zero-downtime approach with rollback plan +- Linked all work back to original spec diff --git a/plugins/notion/skills/notion-spec-to-implementation/examples/ui-component.md b/plugins/notion/skills/notion-spec-to-implementation/examples/ui-component.md new file mode 100644 index 0000000..37d7d95 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/examples/ui-component.md @@ -0,0 +1,68 @@ +# Example: UI Component Implementation + +**User Request**: "Implement the new search component from the design spec" + +## Workflow + +### 1. Find Spec +``` +Notion:search +query: "search component design spec" +``` +Found: "Global Search Redesign Spec" + +### 2. Fetch & Parse Spec +``` +Notion:fetch +``` + +**Requirements**: +- Instant search with debounced API calls +- Keyboard navigation (↑↓ arrows, Enter, Esc) +- Recent searches history +- Result categories (pages, people, files) +- Mobile responsive +- Accessibility (ARIA labels, screen reader) + +### 3. Create Implementation Plan +``` +Notion:notion-create-pages +``` + +**Plan phases**: +- Phase 1: Base component structure +- Phase 2: API integration & debouncing +- Phase 3: Keyboard navigation +- Phase 4: Recent searches +- Phase 5: Polish & accessibility + +### 4. Create Tasks +``` +Notion:notion-create-pages (7 tasks) +``` + +**Tasks**: +1. Create SearchInput component +2. Implement useDebounce hook +3. Add keyboard navigation +4. LocalStorage for recent searches +5. Result categorization UI +6. Accessibility audit +7. Mobile responsive styling + +### 5. Implement & Track +As each task completed, updated status and added progress notes with screenshots and implementation details. + +## Key Outputs + +**Implementation Plan** (linked to design spec) +**7 Component Tasks** (in Engineering Tasks database) +**Progress Updates** (with code snippets and demo links) + +## Success Factors +- Clear component breakdown +- Separated concerns (logic, UI, accessibility) +- Each task had acceptance criteria +- Referenced design spec throughout +- Included accessibility from start, not afterthought +- Tracked progress with visual updates diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/milestone-summary-template.md b/plugins/notion/skills/notion-spec-to-implementation/reference/milestone-summary-template.md new file mode 100644 index 0000000..807cbe2 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/milestone-summary-template.md @@ -0,0 +1,27 @@ +# Milestone Summary Template + +Use this when completing major phases or milestones. + +```markdown +## Phase [N] Complete: [Date] + +### Accomplishments +- [Major item delivered] +- [Major item delivered] + +### Deliverables +- Deliverable 1 +- [Link to PR/deployment] + +### Metrics +- [Relevant metric] +- [Relevant metric] + +### Learnings +- [What went well] +- [What to improve] + +### Next Phase +Starting [Phase name] on [Date] +``` + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/progress-tracking.md b/plugins/notion/skills/notion-spec-to-implementation/reference/progress-tracking.md new file mode 100644 index 0000000..14b0061 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/progress-tracking.md @@ -0,0 +1,458 @@ +# Progress Tracking + +## Update Frequency + +### Daily Updates + +For active implementation work: + +**What to update**: +- Task status if changed +- Add progress note to task +- Update blockers + +**When**: +- End of work day +- After completing significant work +- When encountering blockers + +### Milestone Updates + +For phase/milestone completion: + +**What to update**: +- Mark phase complete in plan +- Add milestone summary +- Update timeline if needed +- Report to stakeholders + +**When**: +- Phase completion +- Major deliverable ready +- Sprint end +- Release + +### Status Change Updates + +For task state transitions: + +**What to update**: +- Task status property +- Add transition note +- Notify relevant people + +**When**: +- Start work (To Do → In Progress) +- Ready for review (In Progress → In Review) +- Complete (In Review → Done) +- Block (Any → Blocked) + +## Progress Note Format + +### Daily Progress Note + +```markdown +## Progress: [Date] + +### Completed +- [Specific accomplishment with details] +- [Specific accomplishment with details] + +### In Progress +- [Current work item] +- Current status: [Percentage or description] + +### Next Steps +1. [Next planned action] +2. [Next planned action] + +### Blockers +- [Blocker description and who/what needed to unblock] +- Or: None + +### Decisions Made +- [Any technical/product decisions] + +### Notes +[Additional context, learnings, issues encountered] +``` + +Example: + +```markdown +## Progress: Oct 14, 2025 + +### Completed +- Implemented user authentication API endpoints (login, logout, refresh) +- Added JWT token generation and validation +- Wrote unit tests for auth service (95% coverage) + +### In Progress +- Frontend login form integration +- Currently: Form submits but need to handle error states + +### Next Steps +1. Complete error handling in login form +2. Add loading states +3. Implement "remember me" functionality + +### Blockers +None + +### Decisions Made +- Using HttpOnly cookies for refresh tokens (more secure than localStorage) +- Session timeout set to 24 hours based on security review + +### Notes +- Found edge case with concurrent login attempts, added to backlog +- Performance of auth check is good (<10ms) +``` + +### Milestone Summary + +```markdown +## Phase [N] Complete: [Date] + +### Overview +[Brief description of what was accomplished in this phase] + +### Completed Tasks +- Task 1 ✅ +- Task 2 ✅ +- Task 3 ✅ + +### Deliverables +- [Deliverable 1]: [Link/description] +- [Deliverable 2]: [Link/description] + +### Key Accomplishments +- [Major achievement] +- [Major achievement] + +### Metrics +- [Relevant metric]: [Value] +- [Relevant metric]: [Value] + +### Challenges Overcome +- [Challenge and how it was solved] + +### Learnings +**What went well**: +- [Success factor] + +**What to improve**: +- [Area for improvement] + +### Impact on Timeline +- On schedule / [X days ahead/behind] +- Reason: [If deviation, explain why] + +### Next Phase +- **Starting**: [Next phase name] +- **Target start date**: [Date] +- **Focus**: [Main objectives] +``` + +## Updating Implementation Plan + +### Progress Indicators + +Update plan page regularly: + +```markdown +## Status Overview + +**Overall Progress**: 45% complete + +### Phase Status +- ✅ Phase 1: Foundation - Complete +- 🔄 Phase 2: Core Features - In Progress (60%) +- ⏳ Phase 3: Integration - Not Started + +### Task Summary +- ✅ Completed: 12 tasks +- 🔄 In Progress: 5 tasks +- 🚧 Blocked: 1 task +- ⏳ Not Started: 8 tasks + +**Last Updated**: [Date] +``` + +### Task Checklist Updates + +Mark completed tasks: + +```markdown +## Implementation Phases + +### Phase 1: Foundation +- [x] Database schema +- [x] API scaffolding +- [x] Auth setup + +### Phase 2: Core Features +- [x] User management +- [ ] Dashboard +- [ ] Reporting +``` + +### Timeline Updates + +Update milestone dates: + +```markdown +## Timeline + +| Milestone | Original | Current | Status | +|-----------|----------|---------|--------| +| Phase 1 | Oct 15 | Oct 14 | ✅ Complete (1 day early) | +| Phase 2 | Oct 30 | Nov 2 | 🔄 In Progress (3 days delay) | +| Phase 3 | Nov 15 | Nov 18 | ⏳ Planned (adjusted) | +| Launch | Nov 20 | Nov 22 | ⏳ Planned (adjusted) | + +**Timeline Status**: Slightly behind due to [reason] +``` + +## Task Status Tracking + +### Status Definitions + +**To Do**: Not started +- Task is ready to begin +- Dependencies met +- Assigned (or available) + +**In Progress**: Actively being worked +- Work has started +- Assigned to someone +- Regular updates expected + +**Blocked**: Cannot proceed +- Dependency not met +- External blocker +- Waiting on decision/resource + +**In Review**: Awaiting review +- Work complete from implementer perspective +- Needs code review, QA, or approval +- Reviewers identified + +**Done**: Complete +- All acceptance criteria met +- Reviewed and approved +- Deployed/delivered + +### Updating Task Status + +When updating: + +``` +1. Update Status property +2. Add progress note explaining change +3. Update related tasks if needed +4. Notify relevant people via comment + +Example: +properties: { "Status": "In Progress" } + +Content update: +## Progress: Oct 14, 2025 +Started implementation. Set up basic structure and wrote initial tests. +``` + +## Blocker Tracking + +### Recording Blockers + +When encountering a blocker: + +```markdown +## Blockers + +### [Date]: [Blocker Description] +**Status**: 🚧 Active +**Impact**: [What's blocked] +**Needed to unblock**: [Action/person/decision needed] +**Owner**: [Who's responsible for unblocking] +**Target resolution**: [Date or timeframe] +``` + +### Resolving Blockers + +When unblocked: + +```markdown +## Blockers + +### [Date]: [Blocker Description] +**Status**: ✅ Resolved on [Date] +**Resolution**: [How it was resolved] +**Impact**: [Any timeline/scope impact] +``` + +### Escalating Blockers + +If blocker needs escalation: + +``` +1. Update blocker status in task +2. Add comment tagging stakeholder +3. Update plan with blocker impact +4. Propose mitigation if possible +``` + +## Metrics Tracking + +### Velocity Tracking + +Track completion rate: + +```markdown +## Velocity + +### Week 1 +- Tasks completed: 8 +- Story points: 21 +- Velocity: Strong + +### Week 2 +- Tasks completed: 6 +- Story points: 18 +- Velocity: Moderate (1 blocker) + +### Week 3 +- Tasks completed: 9 +- Story points: 24 +- Velocity: Strong (blocker resolved) +``` + +### Quality Metrics + +Track quality indicators: + +```markdown +## Quality Metrics + +- Test coverage: 87% +- Code review approval rate: 95% +- Bug count: 3 (2 minor, 1 cosmetic) +- Performance: All targets met +- Security: No issues found +``` + +### Progress Metrics + +Quantitative progress: + +```markdown +## Progress Metrics + +- Requirements implemented: 15/20 (75%) +- Acceptance criteria met: 42/56 (75%) +- Test cases passing: 128/135 (95%) +- Code complete: 80% +- Documentation: 60% +``` + +## Stakeholder Communication + +### Weekly Status Report + +```markdown +## Weekly Status: [Week of Date] + +### Summary +[One paragraph overview of progress and status] + +### This Week's Accomplishments +- [Key accomplishment] +- [Key accomplishment] +- [Key accomplishment] + +### Next Week's Plan +- [Planned work] +- [Planned work] + +### Status +- On track / At risk / Behind schedule +- [If at risk or behind, explain and provide mitigation plan] + +### Blockers & Needs +- [Active blocker or need for help] +- Or: None + +### Risks +- [New or evolving risk] +- Or: None currently identified +``` + +### Executive Summary + +For leadership updates: + +```markdown +## Implementation Status: [Feature Name] + +**Overall Status**: 🟢 On Track / 🟡 At Risk / 🔴 Behind + +**Progress**: [X]% complete + +**Key Updates**: +- [Most important update] +- [Most important update] + +**Timeline**: [Status vs original plan] + +**Risks**: [Top 1-2 risks] + +**Next Milestone**: [Upcoming milestone and date] +``` + +## Automated Progress Tracking + +### Query-Based Status + +Generate status from task database: + +``` +Query task database: +SELECT + "Status", + COUNT(*) as count +FROM "collection://tasks-uuid" +WHERE "Related Tasks" CONTAINS 'plan-page-id' +GROUP BY "Status" + +Generate summary: +- To Do: 8 +- In Progress: 5 +- Blocked: 1 +- In Review: 2 +- Done: 12 + +Overall: 44% complete (12/28 tasks) +``` + +### Timeline Calculation + +Calculate projected completion: + +``` +Average velocity: 6 tasks/week +Remaining tasks: 14 +Projected completion: 2.3 weeks from now + +Compares to target: [On schedule/Behind/Ahead] +``` + +## Best Practices + +1. **Update regularly**: Don't let updates pile up +2. **Be specific**: "Completed login" vs "Made progress" +3. **Quantify progress**: Use percentages, counts, metrics +4. **Note blockers immediately**: Don't wait to report blockers +5. **Link to work**: Reference PRs, deployments, demos +6. **Track decisions**: Document why, not just what +7. **Be honest**: Report actual status, not optimistic status +8. **Update in one place**: Keep implementation plan as source of truth + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/progress-update-template.md b/plugins/notion/skills/notion-spec-to-implementation/reference/progress-update-template.md new file mode 100644 index 0000000..70a2f0b --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/progress-update-template.md @@ -0,0 +1,25 @@ +# Progress Update Template + +Use this to update progress on implementation plans and tasks. + +```markdown +## Progress: [Date] + +### Completed Today +- [Specific item completed] +- [Specific item completed] + +### In Progress +- [Current work item and status] + +### Next Steps +1. [Next action] +2. [Next action] + +### Blockers +- [Blocker description] or None + +### Notes +[Additional context, decisions made, issues encountered] +``` + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/quick-implementation-plan.md b/plugins/notion/skills/notion-spec-to-implementation/reference/quick-implementation-plan.md new file mode 100644 index 0000000..be42908 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/quick-implementation-plan.md @@ -0,0 +1,26 @@ +# Quick Implementation Plan Template + +For simpler features or small changes. + +```markdown +# Implementation: [Feature Name] + +## Spec +Specification + +## Summary +[Quick description] + +## Tasks +- [ ] Task 1 +- [ ] Task 2 +- [ ] Task 3 + +## Timeline +Start: [Date] +Target completion: [Date] + +## Status +[Update as work progresses] +``` + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/spec-parsing.md b/plugins/notion/skills/notion-spec-to-implementation/reference/spec-parsing.md new file mode 100644 index 0000000..1eb148a --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/spec-parsing.md @@ -0,0 +1,383 @@ +# Specification Parsing + +## Finding the Specification + +Before parsing, locate the spec page: + +``` +1. Search for spec: + Notion:search + query: "[Feature Name] spec" or "[Feature Name] specification" + +2. Handle results: + - If found → use page URL/ID + - If multiple → ask user which one + - If not found → ask user for URL/ID + +Example: +Notion:search +query: "User Profile API spec" +query_type: "internal" +``` + +## Reading Specifications + +After finding the spec, fetch it with `Notion:fetch`: + +1. Read the full content +2. Identify key sections +3. Extract structured information +4. Note ambiguities or gaps + +``` +Notion:fetch +id: "spec-page-id-from-search" +``` + +## Common Spec Structures + +### Requirements-Based Spec + +``` +# Feature Spec +## Overview +[Feature description] + +## Requirements +### Functional +- REQ-1: [Requirement] +- REQ-2: [Requirement] + +### Non-Functional +- PERF-1: [Performance requirement] +- SEC-1: [Security requirement] + +## Acceptance Criteria +- AC-1: [Criterion] +- AC-2: [Criterion] +``` + +Extract: +- List of functional requirements +- List of non-functional requirements +- List of acceptance criteria + +### User Story Based Spec + +``` +# Feature Spec +## User Stories +### As a [user type] +I want [goal] +So that [benefit] + +**Acceptance Criteria**: +- [Criterion] +- [Criterion] +``` + +Extract: +- User personas +- Goals/capabilities needed +- Acceptance criteria per story + +### Technical Design Doc + +``` +# Technical Design +## Problem Statement +[Problem description] + +## Proposed Solution +[Solution approach] + +## Architecture +[Architecture details] + +## Implementation Plan +[Implementation approach] +``` + +Extract: +- Problem being solved +- Proposed solution approach +- Architectural decisions +- Implementation guidance + +### Product Requirements Document (PRD) + +``` +# PRD: [Feature] +## Goals +[Business goals] + +## User Needs +[User problems being solved] + +## Features +[Feature list] + +## Success Metrics +[How to measure success] +``` + +Extract: +- Business goals +- User needs +- Feature list +- Success metrics + +## Extraction Strategies + +### Requirement Identification + +Look for: +- "Must", "Should", "Will" statements +- Numbered requirements (REQ-1, etc.) +- User stories (As a... I want...) +- Acceptance criteria sections +- Feature lists + +### Categorization + +Group requirements by: + +**Functional**: What the system does +- User capabilities +- System behaviors +- Data operations + +**Non-Functional**: How the system performs +- Performance targets +- Security requirements +- Scalability needs +- Availability requirements +- Compliance requirements + +**Constraints**: Limitations +- Technical constraints +- Business constraints +- Timeline constraints + +### Priority Extraction + +Identify priority indicators: +- "Critical", "Must have", "P0" +- "Important", "Should have", "P1" +- "Nice to have", "Could have", "P2" +- "Future", "Won't have", "P3" + +Map to implementation phases based on priority. + +## Handling Ambiguity + +### Unclear Requirements + +When requirement is ambiguous: + +```markdown +## Clarifications Needed + +### [Requirement ID/Description] +**Current text**: "[Ambiguous requirement]" +**Question**: [What needs clarification] +**Impact**: [Why this matters for implementation] +**Assumed for now**: [Working assumption if any] +``` + +Create clarification task or add comment to spec. + +### Missing Information + +When critical info is missing: + +```markdown +## Missing Information + +- **[Topic]**: Spec doesn't specify [what's missing] +- **Impact**: Blocks [affected tasks] +- **Action**: Need to [how to resolve] +``` + +### Conflicting Requirements + +When requirements conflict: + +```markdown +## Conflicting Requirements + +**Conflict**: REQ-1 says [X] but REQ-5 says [Y] +**Impact**: [Implementation impact] +**Resolution needed**: [Decision needed] +``` + +## Acceptance Criteria Parsing + +### Explicit Criteria + +Direct acceptance criteria: + +``` +## Acceptance Criteria +- User can log in with email and password +- System sends confirmation email +- Session expires after 24 hours +``` + +Convert to checklist: +- [ ] User can log in with email and password +- [ ] System sends confirmation email +- [ ] Session expires after 24 hours + +### Implicit Criteria + +Derive from requirements: + +``` +Requirement: "Users can upload files up to 100MB" + +Implied acceptance criteria: +- [ ] Files up to 100MB upload successfully +- [ ] Files over 100MB are rejected with error message +- [ ] Progress indicator shows during upload +- [ ] Upload can be cancelled +``` + +### Testable Criteria + +Ensure criteria are testable: + +❌ **Not testable**: "System is fast" +✓ **Testable**: "Page loads in < 2 seconds" + +❌ **Not testable**: "Users like the interface" +✓ **Testable**: "90% of test users complete task successfully" + +## Technical Detail Extraction + +### Architecture Information + +Extract: +- System components +- Data models +- APIs/interfaces +- Integration points +- Technology choices + +### Design Decisions + +Note: +- Technology selections +- Architecture patterns +- Trade-offs made +- Rationale provided + +### Implementation Guidance + +Look for: +- Suggested approach +- Code examples +- Library recommendations +- Best practices mentioned + +## Dependency Identification + +### External Dependencies + +From spec, identify: +- Third-party services required +- External APIs needed +- Infrastructure requirements +- Tool/library dependencies + +### Internal Dependencies + +Identify: +- Other features needed first +- Shared components required +- Team dependencies +- Data dependencies + +### Timeline Dependencies + +Note: +- Hard deadlines +- Milestone dependencies +- Sequencing requirements + +## Scope Extraction + +### In Scope + +What's explicitly included: +- Features to build +- Use cases to support +- Users/personas to serve + +### Out of Scope + +What's explicitly excluded: +- Features deferred +- Use cases not supported +- Edge cases not handled + +### Assumptions + +What's assumed: +- Environment assumptions +- User assumptions +- System state assumptions + +## Risk Identification + +Extract risk information: + +### Technical Risks +- Unproven technology +- Complex integration +- Performance concerns +- Scalability unknowns + +### Business Risks +- Market timing +- Resource availability +- Dependency on others + +### Mitigation Strategies + +Note any mitigation approaches mentioned in spec. + +## Spec Quality Assessment + +Evaluate spec completeness: + +✓ **Good spec**: +- Clear requirements +- Explicit acceptance criteria +- Priorities defined +- Risks identified +- Technical approach outlined + +⚠️ **Incomplete spec**: +- Vague requirements +- Missing acceptance criteria +- Unclear priorities +- No risk analysis +- Technical details absent + +Document gaps and create clarification tasks. + +## Parsing Checklist + +Before creating implementation plan: + +☐ All functional requirements identified +☐ Non-functional requirements noted +☐ Acceptance criteria extracted +☐ Dependencies identified +☐ Risks noted +☐ Ambiguities documented +☐ Technical approach understood +☐ Scope is clear +☐ Priorities are defined + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/standard-implementation-plan.md b/plugins/notion/skills/notion-spec-to-implementation/reference/standard-implementation-plan.md new file mode 100644 index 0000000..556a1d0 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/standard-implementation-plan.md @@ -0,0 +1,146 @@ +# Standard Implementation Plan Template + +Use this template for most feature implementations. + +```markdown +# Implementation Plan: [Feature Name] + +## Overview +[1-2 sentence feature description and business value] + +## Linked Specification +Original Specification + +## Requirements Summary + +### Functional Requirements +- [Requirement 1] +- [Requirement 2] +- [Requirement 3] + +### Non-Functional Requirements +- **Performance**: [Targets] +- **Security**: [Requirements] +- **Scalability**: [Needs] + +### Acceptance Criteria +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [Criterion 3] + +## Technical Approach + +### Architecture +[High-level architectural decisions] + +### Technology Stack +- Backend: [Technologies] +- Frontend: [Technologies] +- Infrastructure: [Technologies] + +### Key Design Decisions +1. **[Decision]**: [Rationale] +2. **[Decision]**: [Rationale] + +## Implementation Phases + +### Phase 1: Foundation (Week 1) +**Goal**: Set up core infrastructure + +**Tasks**: +- [ ] Database schema design +- [ ] API scaffolding +- [ ] Authentication setup + +**Deliverables**: Working API skeleton +**Estimated effort**: 3 days + +### Phase 2: Core Features (Week 2-3) +**Goal**: Implement main functionality + +**Tasks**: +- [ ] Feature A implementation +- [ ] Feature B implementation + +**Deliverables**: Core features working +**Estimated effort**: 1 week + +### Phase 3: Integration & Polish (Week 4) +**Goal**: Complete integration and refinement + +**Tasks**: +- [ ] Frontend integration +- [ ] Testing & QA + +**Deliverables**: Production-ready feature +**Estimated effort**: 1 week + +## Dependencies + +### External Dependencies +- [Dependency 1]: [Status] +- [Dependency 2]: [Status] + +### Internal Dependencies +- [Team/component dependency] + +### Blockers +- [Known blocker] or None currently + +## Risks & Mitigation + +### Risk 1: [Description] +- **Probability**: High/Medium/Low +- **Impact**: High/Medium/Low +- **Mitigation**: [Strategy] + +### Risk 2: [Description] +- **Probability**: High/Medium/Low +- **Impact**: High/Medium/Low +- **Mitigation**: [Strategy] + +## Timeline + +| Milestone | Target Date | Status | +|-----------|-------------|--------| +| Phase 1 Complete | [Date] | ⏳ Planned | +| Phase 2 Complete | [Date] | ⏳ Planned | +| Phase 3 Complete | [Date] | ⏳ Planned | +| Launch | [Date] | ⏳ Planned | + +## Success Criteria + +### Technical Success +- [ ] All acceptance criteria met +- [ ] Performance targets achieved +- [ ] Security requirements satisfied +- [ ] Test coverage > 80% + +### Business Success +- [ ] [Business metric 1] +- [ ] [Business metric 2] + +## Resources + +### Documentation +- Design Doc +- API Spec + +### Related Work +- Related Feature + +## Progress Tracking + +[This section updated regularly] + +### Phase Status +- Phase 1: ⏳ Not Started +- Phase 2: ⏳ Not Started +- Phase 3: ⏳ Not Started + +**Overall Progress**: 0% complete + +### Latest Update: [Date] +[Brief status update] +``` + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/task-creation-template.md b/plugins/notion/skills/notion-spec-to-implementation/reference/task-creation-template.md new file mode 100644 index 0000000..be0b869 --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/task-creation-template.md @@ -0,0 +1,34 @@ +# Task Creation Template + +When creating tasks from spec. + +```markdown +# [Task Name] + +## Context +Part of implementation for Feature Spec + +Implementation plan: Implementation Plan + +## Description +[What needs to be done] + +## Acceptance Criteria +- [ ] [Criterion 1] +- [ ] [Criterion 2] + +## Technical Details +[Technical approach or notes] + +## Dependencies +- Blocked by: [Task] or None +- Blocks: [Task] or None + +## Resources +- [Link to design] +- [Link to related code] + +## Progress +[To be updated during implementation] +``` + diff --git a/plugins/notion/skills/notion-spec-to-implementation/reference/task-creation.md b/plugins/notion/skills/notion-spec-to-implementation/reference/task-creation.md new file mode 100644 index 0000000..afe28da --- /dev/null +++ b/plugins/notion/skills/notion-spec-to-implementation/reference/task-creation.md @@ -0,0 +1,441 @@ +# Task Creation from Specs + +## Finding the Task Database + +Before creating tasks, locate the task database: + +``` +1. Search for task database: + Notion:search + query: "Tasks" or "Task Management" or "[Project] Tasks" + +2. Fetch database schema: + Notion:fetch + id: "database-id-from-search" + +3. Identify data source: + - Look for tags + - Extract collection ID for parent parameter + +4. Note schema: + - Required properties + - Property types and options + - Relation properties for linking + +Example: +Notion:search +query: "Engineering Tasks" +query_type: "internal" + +Notion:fetch +id: "tasks-database-id" +``` + +Result: `collection://abc-123-def` for use as parent + +## Task Breakdown Strategy + +### Size Guidelines + +**Good task size**: +- Completable in 1-2 days +- Single clear deliverable +- Independently testable +- Minimal dependencies + +**Too large**: +- Takes > 3 days +- Multiple deliverables +- Many dependencies +- Break down further + +**Too small**: +- Takes < 2 hours +- Too granular +- Group with related work + +### Granularity by Phase + +**Early phases**: Larger tasks acceptable +- "Design database schema" +- "Set up API structure" + +**Middle phases**: Medium-sized tasks +- "Implement user authentication" +- "Build dashboard UI" + +**Late phases**: Smaller, precise tasks +- "Fix validation bug in form" +- "Add loading state to button" + +## Task Creation Pattern + +For each requirement or work item: + +``` +1. Identify the work +2. Determine task size +3. Create task in database +4. Set properties +5. Write task description +6. Link to spec/plan +``` + +### Creating Task + +``` +Use Notion:notion-create-pages: + +parent: { + type: "data_source_id", + data_source_id: "collection://tasks-db-uuid" +} + +properties: { + "[Title Property]": "Task: [Clear task name]", + "Status": "To Do", + "Priority": "[High/Medium/Low]", + "[Project/Related]": ["spec-page-id", "plan-page-id"], + "Assignee": "[Person]" (if known), + "date:Due Date:start": "[Date]" (if applicable), + "date:Due Date:is_datetime": 0 +} + +content: "[Task description using template]" +``` + +## Task Description Template + +```markdown +# [Task Name] + +## Context +Implementation task for Feature Spec + +Part of Implementation Plan - Phase [N] + +## Objective +[What this task accomplishes] + +## Requirements +Based on spec requirements: +- [Relevant requirement 1] +- [Relevant requirement 2] + +## Acceptance Criteria +- [ ] [Specific, testable criterion] +- [ ] [Specific, testable criterion] +- [ ] [Specific, testable criterion] + +## Technical Approach +[Suggested implementation approach] + +### Components Affected +- [Component 1] +- [Component 2] + +### Key Decisions +- [Decision point 1] +- [Decision point 2] + +## Dependencies + +### Blocked By +- Prerequisite Task or None + +### Blocks +- Dependent Task or None + +## Resources +- [Link to design mockup] +- [Link to API spec] +- [Link to relevant code] + +## Estimated Effort +[Time estimate] + +## Progress +[To be updated during implementation] +``` + +## Task Types + +### Infrastructure/Setup Tasks + +``` +Title: "Setup: [What's being set up]" +Examples: +- "Setup: Configure database connection pool" +- "Setup: Initialize authentication middleware" +- "Setup: Create CI/CD pipeline" + +Focus: Getting environment/tooling ready +``` + +### Feature Implementation Tasks + +``` +Title: "Implement: [Feature name]" +Examples: +- "Implement: User login flow" +- "Implement: File upload functionality" +- "Implement: Dashboard widget" + +Focus: Building specific functionality +``` + +### Integration Tasks + +``` +Title: "Integrate: [What's being integrated]" +Examples: +- "Integrate: Connect frontend to API" +- "Integrate: Add payment provider" +- "Integrate: Link user profile to dashboard" + +Focus: Connecting components +``` + +### Testing Tasks + +``` +Title: "Test: [What's being tested]" +Examples: +- "Test: Write unit tests for auth service" +- "Test: E2E testing for checkout flow" +- "Test: Performance testing for API" + +Focus: Validation and quality assurance +``` + +### Documentation Tasks + +``` +Title: "Document: [What's being documented]" +Examples: +- "Document: API endpoints" +- "Document: Setup instructions" +- "Document: Architecture decisions" + +Focus: Creating documentation +``` + +### Bug Fix Tasks + +``` +Title: "Fix: [Bug description]" +Examples: +- "Fix: Login error on Safari" +- "Fix: Memory leak in image processing" +- "Fix: Race condition in payment flow" + +Focus: Resolving issues +``` + +### Refactoring Tasks + +``` +Title: "Refactor: [What's being refactored]" +Examples: +- "Refactor: Extract auth logic to service" +- "Refactor: Optimize database queries" +- "Refactor: Simplify component hierarchy" + +Focus: Code quality improvement +``` + +## Sequencing Tasks + +### Critical Path + +Identify must-happen-first tasks: + +``` +1. Database schema +2. API foundation +3. Core business logic +4. Frontend integration +5. Testing +6. Deployment +``` + +### Parallel Tracks + +Tasks that can happen simultaneously: + +``` +Track A: Backend development +- API endpoints +- Business logic +- Database operations + +Track B: Frontend development +- UI components +- State management +- Routing + +Track C: Infrastructure +- CI/CD setup +- Monitoring +- Documentation +``` + +### Phase-Based Sequencing + +Group by implementation phase: + +``` +Phase 1 (Foundation): +- Setup tasks +- Infrastructure tasks + +Phase 2 (Core): +- Feature implementation tasks +- Integration tasks + +Phase 3 (Polish): +- Testing tasks +- Documentation tasks +- Optimization tasks +``` + +## Priority Assignment + +### P0/Critical +- Blocks everything else +- Core functionality +- Security requirements +- Data integrity + +### P1/High +- Important features +- User-facing functionality +- Performance requirements + +### P2/Medium +- Nice-to-have features +- Optimizations +- Minor improvements + +### P3/Low +- Future enhancements +- Edge case handling +- Cosmetic improvements + +## Estimation + +### Story Points + +If using story points: +- 1 point: Few hours +- 2 points: Half day +- 3 points: Full day +- 5 points: 2 days +- 8 points: 3-4 days (consider breaking down) + +### Time Estimates + +Direct time estimates: +- 2-4 hours: Small task +- 1 day: Medium task +- 2 days: Large task +- 3+ days: Break down further + +### Estimation Factors + +Consider: +- Complexity +- Unknowns +- Dependencies +- Testing requirements +- Documentation needs + +## Task Relationships + +### Parent Task Pattern + +For large features: + +``` +Parent: "Feature: User Authentication" +Children: +- "Setup: Configure auth library" +- "Implement: Login flow" +- "Implement: Password reset" +- "Test: Auth functionality" +``` + +### Dependency Chain Pattern + +For sequential work: + +``` +Task A: "Design database schema" +↓ (blocks) +Task B: "Implement data models" +↓ (blocks) +Task C: "Create API endpoints" +↓ (blocks) +Task D: "Integrate with frontend" +``` + +### Related Tasks Pattern + +For parallel work: + +``` +Central: "Feature: Dashboard" +Related: +- "Backend API for dashboard data" +- "Frontend dashboard component" +- "Dashboard data caching" +``` + +## Bulk Task Creation + +When creating many tasks: + +``` +For each work item in breakdown: + 1. Determine task properties + 2. Create task page + 3. Link to spec/plan + 4. Set relationships + +Then: + 1. Update plan with task links + 2. Review sequencing + 3. Assign tasks (if known) +``` + +## Task Naming Conventions + +**Be specific**: +✓ "Implement user login with email/password" +✗ "Add login" + +**Include context**: +✓ "Dashboard: Add revenue chart widget" +✗ "Add chart" + +**Use action verbs**: +- Implement, Build, Create +- Integrate, Connect, Link +- Fix, Resolve, Debug +- Test, Validate, Verify +- Document, Write, Update +- Refactor, Optimize, Improve + +## Validation Checklist + +Before finalizing tasks: + +☐ Each task has clear objective +☐ Acceptance criteria are testable +☐ Dependencies identified +☐ Appropriate size (1-2 days) +☐ Priority assigned +☐ Linked to spec/plan +☐ Proper sequencing +☐ Resources noted + diff --git a/plugins/sentry/.codex-plugin/plugin.json b/plugins/sentry/.codex-plugin/plugin.json new file mode 100644 index 0000000..9d35e40 --- /dev/null +++ b/plugins/sentry/.codex-plugin/plugin.json @@ -0,0 +1,38 @@ +{ + "name": "sentry", + "version": "0.1.0", + "description": "Inspect recent issues and events in Sentry from Codex.", + "author": { + "name": "OpenAI", + "email": "support@openai.com", + "url": "https://openai.com/" + }, + "homepage": "https://sentry.io/", + "repository": "https://github.com/openai/plugins", + "license": "MIT", + "keywords": [ + "sentry", + "observability", + "triage" + ], + "skills": "./skills/", + "interface": { + "displayName": "Sentry", + "shortDescription": "Inspect recent Sentry issues and events", + "longDescription": "Use Sentry skills to inspect recent issues, review events, and summarize production errors with a read-only workflow.", + "developerName": "OpenAI", + "category": "Productivity", + "capabilities": [ + "Interactive", + "Write" + ], + "websiteURL": "https://sentry.io/", + "privacyPolicyURL": "https://sentry.io/privacy/", + "termsOfServiceURL": "https://sentry.io/terms/", + "defaultPrompt": "Use Sentry skills to inspect recent production issues, summarize the top errors, and pull issue details without exposing tokens or raw stack traces.", + "composerIcon": "./assets/sentry-small.svg", + "logo": "./assets/sentry.png", + "screenshots": [], + "brandColor": "#362D59" + } +} diff --git a/plugins/sentry/assets/sentry-small.svg b/plugins/sentry/assets/sentry-small.svg new file mode 100644 index 0000000..6b975c1 --- /dev/null +++ b/plugins/sentry/assets/sentry-small.svg @@ -0,0 +1,3 @@ + diff --git a/plugins/sentry/assets/sentry.png b/plugins/sentry/assets/sentry.png new file mode 100644 index 0000000..9f8ddce Binary files /dev/null and b/plugins/sentry/assets/sentry.png differ diff --git a/plugins/sentry/skills/sentry/LICENSE.txt b/plugins/sentry/skills/sentry/LICENSE.txt new file mode 100644 index 0000000..13e25df --- /dev/null +++ b/plugins/sentry/skills/sentry/LICENSE.txt @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of + any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don\'t include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/plugins/sentry/skills/sentry/SKILL.md b/plugins/sentry/skills/sentry/SKILL.md new file mode 100644 index 0000000..a227fd6 --- /dev/null +++ b/plugins/sentry/skills/sentry/SKILL.md @@ -0,0 +1,128 @@ +--- +name: "sentry" +description: "Use when the user asks to inspect Sentry issues or events, summarize recent production errors, or pull basic Sentry health data via the Sentry API; perform read-only queries with the bundled script and require `SENTRY_AUTH_TOKEN`." +--- + + +# Sentry (Read-only Observability) + +## Quick start + +- If not already authenticated, ask the user to provide a valid `SENTRY_AUTH_TOKEN` (read-only scopes such as `project:read`, `event:read`) or to log in and create one before running commands. +- Set `SENTRY_AUTH_TOKEN` as an env var. +- Optional defaults: `SENTRY_ORG`, `SENTRY_PROJECT`, `SENTRY_BASE_URL`. +- Defaults: org/project `{your-org}`/`{your-project}`, time range `24h`, environment `prod`, limit 20 (max 50). +- Always call the Sentry API (no heuristics, no caching). + +If the token is missing, give the user these steps: +1. Create a Sentry auth token: https://sentry.io/settings/account/api/auth-tokens/ +2. Create a token with read-only scopes such as `project:read`, `event:read`, and `org:read`. +3. Set `SENTRY_AUTH_TOKEN` as an environment variable in their system. +4. Offer to guide them through setting the environment variable for their OS/shell if needed. +- Never ask the user to paste the full token in chat. Ask them to set it locally and confirm when ready. + +## Core tasks (use bundled script) + +Use `scripts/sentry_api.py` for deterministic API calls. It handles pagination and retries once on transient errors. + +## Skill path (set once) + +```bash +export CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" +export SENTRY_API="$CODEX_HOME/skills/sentry/scripts/sentry_api.py" +``` + +User-scoped skills install under `$CODEX_HOME/skills` (default: `~/.codex/skills`). + +### 1) List issues (ordered by most recent) + +```bash +python3 "$SENTRY_API" \ + --org {your-org} \ + --project {your-project} \ + list-issues \ + --environment prod \ + --time-range 24h \ + --limit 20 \ + --query "is:unresolved" +``` + +### 2) Resolve an issue short ID to issue ID + +```bash +python3 "$SENTRY_API" \ + --org {your-org} \ + --project {your-project} \ + list-issues \ + --query "ABC-123" \ + --limit 1 +``` + +Use the returned `id` for issue detail or events. + +### 3) Issue detail + +```bash +python3 "$SENTRY_API" \ + --org {your-org} \ + issue-detail \ + 1234567890 +``` + +### 4) Issue events + +```bash +python3 "$SENTRY_API" \ + --org {your-org} \ + issue-events \ + 1234567890 \ + --environment prod \ + --time-range 24h \ + --limit 20 +``` + +### 5) Event detail (no stack traces by default) + +```bash +python3 "$SENTRY_API" \ + --org {your-org} \ + --project {your-project} \ + event-detail \ + abcdef1234567890 +``` + +## API requirements + +Always use these endpoints (GET only): + +- List issues: `/api/0/projects/{org_slug}/{project_slug}/issues/` +- Issue detail: `/api/0/organizations/{org_slug}/issues/{issue_id}/` +- Events for issue: `/api/0/organizations/{org_slug}/issues/{issue_id}/events/` +- Event detail: `/api/0/projects/{org_slug}/{project_slug}/events/{event_id}/` + +## Inputs and defaults + +- `org_slug`: default to `{your-org}` (required for issue detail, issue events, and event detail). +- `project_slug`: default to `{your-project}` (required for list issues and event detail). +- `time_range`: default `24h` (pass as `statsPeriod` for list issues and issue events). +- `environment`: default `prod` (used by list issues and issue events). +- `limit`: default 20, max 50 (paginate until limit reached). +- `search_query`: optional `query` parameter. +- `issue_short_id`: resolve via list-issues query first. + +## Output formatting rules + +- Issue list: show title, short_id, status, first_seen, last_seen, count, environments, top_tags; order by most recent. +- Event detail: include culprit, timestamp, environment, release, url. +- If no results, state explicitly. +- Redact PII in output (emails, IPs). Do not print raw stack traces. +- Never echo auth tokens. + +## Golden test inputs + +- Org: `{your-org}` +- Project: `{your-project}` +- Issue short ID: `{ABC-123}` + +Example prompt: “List the top 10 open issues for prod in the last 24h.” +Expected: ordered list with titles, short IDs, counts, last seen. diff --git a/plugins/sentry/skills/sentry/agents/openai.yaml b/plugins/sentry/skills/sentry/agents/openai.yaml new file mode 100644 index 0000000..834c318 --- /dev/null +++ b/plugins/sentry/skills/sentry/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Sentry (Read-only Observability)" + short_description: "Read-only Sentry observability" + icon_small: "./assets/sentry-small.svg" + icon_large: "./assets/sentry.png" + default_prompt: "Investigate this issue in read-only Sentry data and report likely root cause, impact, and next steps." diff --git a/plugins/sentry/skills/sentry/assets/sentry-small.svg b/plugins/sentry/skills/sentry/assets/sentry-small.svg new file mode 100644 index 0000000..6b975c1 --- /dev/null +++ b/plugins/sentry/skills/sentry/assets/sentry-small.svg @@ -0,0 +1,3 @@ + diff --git a/plugins/sentry/skills/sentry/assets/sentry.png b/plugins/sentry/skills/sentry/assets/sentry.png new file mode 100644 index 0000000..9f8ddce Binary files /dev/null and b/plugins/sentry/skills/sentry/assets/sentry.png differ diff --git a/plugins/sentry/skills/sentry/scripts/sentry_api.py b/plugins/sentry/skills/sentry/scripts/sentry_api.py new file mode 100755 index 0000000..3fa7721 --- /dev/null +++ b/plugins/sentry/skills/sentry/scripts/sentry_api.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +DEFAULT_BASE_URL = "https://sentry.io" +DEFAULT_ORG = "your-org" +DEFAULT_PROJECT = "your-project" +MAX_LIMIT = 50 + +EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") +IP_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b") + + +def redact_string(value): + value = EMAIL_RE.sub("[REDACTED_EMAIL]", value) + value = IP_RE.sub("[REDACTED_IP]", value) + return value + + +def redact_data(value): + if isinstance(value, str): + return redact_string(value) + if isinstance(value, list): + return [redact_data(item) for item in value] + if isinstance(value, dict): + redacted = {} + for key, item in value.items(): + if key.lower() in {"email", "ip", "ip_address"}: + redacted[key] = "[REDACTED]" + else: + redacted[key] = redact_data(item) + return redacted + return value + + +def next_cursor(link_header): + if not link_header: + return None + for part in link_header.split(","): + if 'rel="next"' in part and 'results="true"' in part: + match = re.search(r'cursor="([^"]+)"', part) + if match: + return match.group(1) + return None + + +def request_json(url, token, retries=1): + req = Request(url) + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Accept", "application/json") + + attempt = 0 + while True: + try: + with urlopen(req) as resp: + body = resp.read().decode("utf-8") + data = json.loads(body) if body else None + return data, resp.headers + except HTTPError as err: + body = err.read().decode("utf-8", "ignore") + if attempt < retries and (err.code >= 500 or err.code == 429): + attempt += 1 + time.sleep(1) + continue + raise RuntimeError(f"HTTP {err.code} for {url}: {body or 'request failed'}") from err + except URLError as err: + if attempt < retries: + attempt += 1 + time.sleep(1) + continue + raise RuntimeError(f"Network error for {url}: {err.reason}") from err + + +def build_url(base_url, path, params=None): + base = base_url.rstrip("/") + url = f"{base}{path}" + if params: + url = f"{url}?{urlencode(params, doseq=True)}" + return url + + +def paged_get(base_url, path, params, token, limit): + results = [] + cursor = None + while len(results) < limit: + page_params = dict(params) + page_params["per_page"] = min(MAX_LIMIT, limit - len(results)) + if cursor: + page_params["cursor"] = cursor + url = build_url(base_url, path, page_params) + data, headers = request_json(url, token) + if not data: + break + results.extend(data) + cursor = next_cursor(headers.get("Link")) + if not cursor: + break + return results[:limit] + + +def require_org_project(org, project): + if org == DEFAULT_ORG or project == DEFAULT_PROJECT: + raise RuntimeError( + "Missing org/project. Set SENTRY_ORG and SENTRY_PROJECT or pass --org/--project." + ) + + +def require_org(org): + if org == DEFAULT_ORG: + raise RuntimeError("Missing org. Set SENTRY_ORG or pass --org.") + + +def handle_list_issues(args, token, base_url): + require_org_project(args.org, args.project) + limit = min(args.limit, MAX_LIMIT) + params = { + "statsPeriod": args.time_range, + "environment": args.environment, + } + if args.query: + params["query"] = args.query + + path = f"/api/0/projects/{args.org}/{args.project}/issues/" + issues = paged_get(base_url, path, params, token, limit) + return issues + + +def handle_issue_detail(args, token, base_url): + require_org(args.org) + path = f"/api/0/organizations/{args.org}/issues/{args.issue_id}/" + url = build_url(base_url, path) + data, _ = request_json(url, token) + return data + + +def handle_issue_events(args, token, base_url): + require_org(args.org) + limit = min(args.limit, MAX_LIMIT) + params = { + "statsPeriod": args.time_range, + "environment": args.environment, + } + if args.query: + params["query"] = args.query + + path = f"/api/0/organizations/{args.org}/issues/{args.issue_id}/events/" + events = paged_get(base_url, path, params, token, limit) + return events + + +def handle_event_detail(args, token, base_url): + require_org_project(args.org, args.project) + path = f"/api/0/projects/{args.org}/{args.project}/events/{args.event_id}/" + url = build_url(base_url, path) + data, _ = request_json(url, token) + if data and not args.include_entries: + data = dict(data) + data.pop("entries", None) + return data + + +def build_parser(): + parser = argparse.ArgumentParser( + description="Read-only Sentry API helper for issues and events" + ) + parser.add_argument( + "--base-url", + default=os.environ.get("SENTRY_BASE_URL", DEFAULT_BASE_URL), + help="Sentry base URL (default: https://sentry.io)", + ) + parser.add_argument( + "--org", + default=os.environ.get("SENTRY_ORG", DEFAULT_ORG), + help="Sentry org slug", + ) + parser.add_argument( + "--project", + default=os.environ.get("SENTRY_PROJECT", DEFAULT_PROJECT), + help="Sentry project slug", + ) + parser.add_argument( + "--no-redact", + action="store_true", + help="Do not redact PII in output", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + list_issues = subparsers.add_parser("list-issues", help="List issues") + list_issues.add_argument("--time-range", default="24h") + list_issues.add_argument("--environment", default="prod") + list_issues.add_argument("--query", default="") + list_issues.add_argument("--limit", type=int, default=20) + + issue_detail = subparsers.add_parser("issue-detail", help="Issue detail") + issue_detail.add_argument("issue_id") + + issue_events = subparsers.add_parser("issue-events", help="Issue events") + issue_events.add_argument("issue_id") + issue_events.add_argument("--time-range", default="24h") + issue_events.add_argument("--environment", default="prod") + issue_events.add_argument("--query", default="") + issue_events.add_argument("--limit", type=int, default=20) + + event_detail = subparsers.add_parser("event-detail", help="Event detail") + event_detail.add_argument("event_id") + event_detail.add_argument( + "--include-entries", + action="store_true", + help="Include event entries (may contain stack traces)", + ) + + return parser + + +def main(): + parser = build_parser() + args = parser.parse_args() + + token = os.environ.get("SENTRY_AUTH_TOKEN") + if not token: + raise RuntimeError("Missing SENTRY_AUTH_TOKEN env var.") + + base_url = args.base_url + + if args.command == "list-issues": + data = handle_list_issues(args, token, base_url) + elif args.command == "issue-detail": + data = handle_issue_detail(args, token, base_url) + elif args.command == "issue-events": + data = handle_issue_events(args, token, base_url) + elif args.command == "event-detail": + data = handle_event_detail(args, token, base_url) + else: + raise RuntimeError(f"Unknown command: {args.command}") + + if not args.no_redact: + data = redact_data(data) + + print(json.dumps(data, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except RuntimeError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1)