Get Started
Examples
Concepts
Resources
Projects
Integrations
API Reference
Object Detection
copy markdown
Detect and identify objects from an input image. It locates bounding boxes, classify objects, and return structured metadata describing each detected entity.
- Accurate bounding boxes and segmentation masks for each object
- Support for a wide range of industries, construction, retail, healthcare, biology, and more
- Computer GUI detection
Real time object detection of an image
Interfaze SDK
Vercel AI SDK
LangChain SDK
typescript
python
import { responseFormat } from "interfaze";
import { z } from "zod";
const DetectionSchema = z.object({
objects: z.array(
z.object({
name: z.string().describe("describe the object in the image"),
top_left_x: z.number(),
top_left_y: z.number(),
bottom_right_x: z.number(),
bottom_right_y: z.number(),
})
),
texts: z
.array(
z.object({
text: z.string(),
top_left_x: z.number(),
top_left_y: z.number(),
bottom_right_x: z.number(),
bottom_right_y: z.number(),
})
)
.describe("any alphabetic characters text in the image"),
});
const response = await interfaze.chat.completions.create({
messages: [
{
role: "user",
content: [
{ type: "text", text: "Get the position of the crane in the image and any text" },
{
type: "image_url",
image_url: {
url: "https://r2public.jigsawstack.com/interfaze/examples/construction.png",
},
},
],
},
],
response_format: responseFormat(z.toJSONSchema(DetectionSchema), "detection_schema"),
});
console.log(JSON.parse(response.choices[0]?.message.content ?? "{}"));
console.log("Object Detection Results:", response.precontext?.[0]?.result);Bounding boxes mapped to the image
JSON output
object contains the extracted information defined in the schema. precontext contains the raw metadata such as bounding boxes and confidence scores.
{
"object": {
"objects": [
{
"name": "crane",
"top_left_x": 630,
"top_left_y": 139,
"bottom_right_x": 769,
"bottom_right_y": 225
}
],
"texts": [
{
"text": "09-12-2020 Sat 21:26:51",
"top_left_x": 218,
"top_left_y": 88,
"bottom_right_x": 623,
"bottom_right_y": 117
},
{
"text": "WRW tower",
"top_left_x": 1069,
"top_left_y": 720,
"bottom_right_x": 1230,
"bottom_right_y": 751
}
]
},
"response": {
"id": "interfaze-1775001605439",
"modelId": "interfaze-beta",
"body": {
"id": "interfaze-1775001605439",
"object": "chat.completion",
"model": "interfaze-beta",
"usage": {
"prompt_tokens": 4903,
"completion_tokens": 8566,
"total_tokens": 13469
},
"precontext": [
{
"name": "object_detection",
"result": {
"detected_objects": [
{
"bounds": {
"top_left": {
"x": 630,
"y": 139
},
"top_right": {
"x": 769,
"y": 139
},
"bottom_left": {
"x": 630,
"y": 225
},
"bottom_right": {
"x": 769,
"y": 225
},
"width": 139,
"height": 86
},
"label": "crane"
}
],
"gui_elements": [
{
"type": "text",
"bounds": {
"top_left": {
"x": 1140,
"y": 722
},
"top_right": {
"x": 1232,
"y": 722
},
"bottom_left": {
"x": 1140,
"y": 752
},
"bottom_right": {
"x": 1232,
"y": 752
},
"width": 92,
"height": 30
},
"interactivity": false,
"content": "tower"
}
]
}
},
{
"name": "ocr",
"result": {
"extracted_text": "09-12-2020 Sat 21:26:51\nWRW tower",
"sections": [
{
"text": "09-12-2020 Sat 21:26:51\nWRW tower",
"lines": [
{
"text": "09-12-2020 Sat 21:26:51",
"bounds": {
"top_left": {
"x": 218,
"y": 88
},
"top_right": {
"x": 623,
"y": 87
},
"bottom_right": {
"x": 623,
"y": 117
},
"bottom_left": {
"x": 218,
"y": 118
},
"width": 405,
"height": 30
},
"average_confidence": 0.99,
"words": [
{
"text": "09-12-2020",
"bounds": {
"top_left": {
"x": 219,
"y": 90
},
"top_right": {
"x": 392,
"y": 88
},
"bottom_right": {
"x": 391,
"y": 117
},
"bottom_left": {
"x": 219,
"y": 117
},
"width": 172.5,
"height": 28
},
"confidence": 0.99
},
{
"text": "Sat",
"bounds": {
"top_left": {
"x": 413,
"y": 88
},
"top_right": {
"x": 464,
"y": 88
},
"bottom_right": {
"x": 463,
"y": 117
},
"bottom_left": {
"x": 413,
"y": 117
},
"width": 50.5,
"height": 29
},
"confidence": 1
},
{
"text": "21:26:51",
"bounds": {
"top_left": {
"x": 483,
"y": 88
},
"top_right": {
"x": 622,
"y": 87
},
"bottom_right": {
"x": 622,
"y": 118
},
"bottom_left": {
"x": 483,
"y": 117
},
"width": 139,
"height": 30
},
"confidence": 0.99
}
]
},
{
"text": "WRW tower",
"bounds": {
"top_left": {
"x": 1069,
"y": 720
},
"top_right": {
"x": 1230,
"y": 721
},
"bottom_right": {
"x": 1230,
"y": 751
},
"bottom_left": {
"x": 1069,
"y": 750
},
"width": 161,
"height": 30
},
"average_confidence": 0.99,
"words": [
{
"text": "WRW",
"bounds": {
"top_left": {
"x": 1070,
"y": 721
},
"top_right": {
"x": 1120,
"y": 721
},
"bottom_right": {
"x": 1120,
"y": 750
},
"bottom_left": {
"x": 1070,
"y": 751
},
"width": 50,
"height": 29.5
},
"confidence": 0.99
},
{
"text": "tower",
"bounds": {
"top_left": {
"x": 1145,
"y": 722
},
"top_right": {
"x": 1227,
"y": 723
},
"bottom_right": {
"x": 1227,
"y": 750
},
"bottom_left": {
"x": 1145,
"y": 750
},
"width": 82,
"height": 27.5
},
"confidence": 0.99
}
]
}
]
}
],
"width": 1400,
"height": 789
}
}
]
}
},
"finishReason": "stop",
"usage": {
"inputTokens": 4903,
"outputTokens": 8566,
"totalTokens": 13469
}
}Run object detection task with raw output
Running object detection as a single task with <task>object_detection</task> in the system message makes it cheaper and faster with a fixed structured output that's pre-defined.
Learn more about running a task.
Interfaze SDK
Vercel AI SDK
LangChain SDK
typescript
python
const result = await interfaze.tasks.objectDetection("https://r2public.jigsawstack.com/interfaze/examples/construction.png");
console.log(result);JSON output
{
"object": {
"name": "object_detection",
"result": {
"detected_objects": [
{
"bounds": {
"top_left": { "x": 630, "y": 139 },
"top_right": { "x": 769, "y": 139 },
"bottom_left": { "x": 630, "y": 225 },
"bottom_right": { "x": 769, "y": 225 },
"width": 139,
"height": 86
},
"label": "crane"
}
],
"gui_elements": [
{
"type": "text",
"bounds": {
"top_left": { "x": 1140, "y": 722 },
"top_right": { "x": 1232, "y": 722 },
"bottom_left": { "x": 1140, "y": 752 },
"bottom_right": { "x": 1232, "y": 752 },
"width": 92,
"height": 30
},
"interactivity": false,
"content": "tower"
}
]
}
}
}