# Object Detection

URL: https://interfaze.ai/docs/vision/object-detection

Detect and identify objects from an input image. It locates bounding boxes, classify objects, and return structured metadata describing each detected entity.

- Accurate bounding boxes and segmentation masks for each object
- Support for a wide range of industries, construction, retail, healthcare, biology, and more
- Computer GUI detection

## Real time object detection of an image

**Interfaze SDK · typescript**

```typescript
import { responseFormat } from "interfaze";
import { z } from "zod";

const DetectionSchema = z.object({
	objects: z.array(
		z.object({
			name: z.string().describe("describe the object in the image"),
			top_left_x: z.number(),
			top_left_y: z.number(),
			bottom_right_x: z.number(),
			bottom_right_y: z.number(),
		})
	),
	texts: z
		.array(
			z.object({
				text: z.string(),
				top_left_x: z.number(),
				top_left_y: z.number(),
				bottom_right_x: z.number(),
				bottom_right_y: z.number(),
			})
		)
		.describe("any alphabetic characters text in the image"),
});

const response = await interfaze.chat.completions.create({
	messages: [
		{
			role: "user",
			content: [
				{ type: "text", text: "Get the position of the crane in the image and any text" },
				{
					type: "image_url",
					image_url: {
						url: "https://r2public.jigsawstack.com/interfaze/examples/construction.png",
					},
				},
			],
		},
	],
	response_format: responseFormat(z.toJSONSchema(DetectionSchema), "detection_schema"),
});

console.log(JSON.parse(response.choices[0]?.message.content ?? "{}"));

console.log("Object Detection Results:", response.precontext?.[0]?.result);
```

**Vercel AI SDK · typescript**

```typescript
import { generateObject } from "ai";
import { z } from "zod";

const DetectionSchema = z.object({
	objects: z.array(
		z.object({
			name: z.string().describe("describe the object in the image"),
			top_left_x: z.number(),
			top_left_y: z.number(),
			bottom_right_x: z.number(),
			bottom_right_y: z.number(),
		})
	),
	texts: z
		.array(
			z.object({
				text: z.string(),
				top_left_x: z.number(),
				top_left_y: z.number(),
				bottom_right_x: z.number(),
				bottom_right_y: z.number(),
			})
		)
		.describe("any alphabetic characters text in the image"),
});

const { object, providerMetadata } = await generateObject({
	model: interfaze("interfaze-beta"),
	schema: DetectionSchema,
	messages: [
		{
			role: "user",
			content: [
				{ type: "text", text: "Get the position of the crane in the image and any text" },
				{
					type: "image",
					mediaType: "image/png",
					image: "https://r2public.jigsawstack.com/interfaze/examples/construction.png",
				},
			],
		},
	],
});

console.log(object);
console.log("Object Detection Results:", providerMetadata?.interfaze?.precontext?.[0]?.result);
```

**LangChain SDK · typescript**

```typescript
import { z } from "zod";

const DetectionSchema = z.object({
	objects: z.array(
		z.object({
			name: z.string().describe("describe the object in the image"),
			top_left_x: z.number(),
			top_left_y: z.number(),
			bottom_right_x: z.number(),
			bottom_right_y: z.number(),
		})
	),
	texts: z
		.array(
			z.object({
				text: z.string(),
				top_left_x: z.number(),
				top_left_y: z.number(),
				bottom_right_x: z.number(),
				bottom_right_y: z.number(),
			})
		)
		.describe("any alphabetic characters text in the image"),
});

const structuredModel = interfaze.withStructuredOutput(DetectionSchema, { includeRaw: true });

const response = await structuredModel.invoke([
	{
		role: "user",
		content: [
			{ type: "text", text: "Get the position of the crane in the image and any text" },
			{
				type: "image_url",
				image_url: {
					url: "https://r2public.jigsawstack.com/interfaze/examples/construction.png",
				},
			},
		],
	},
]);

console.log(response.parsed);

console.log("Object Detection Results:", response.raw.response_metadata.precontext?.[0]?.result);
```

**Interfaze SDK · python**

```python
from pydantic import BaseModel, Field
from typing import List

class DetectedObject(BaseModel):
    name: str = Field(..., description="describe the object in the image")
    top_left_x: float
    top_left_y: float
    bottom_right_x: float
    bottom_right_y: float

class DetectedText(BaseModel):
    text: str
    top_left_x: float
    top_left_y: float
    bottom_right_x: float
    bottom_right_y: float

class DetectionSchema(BaseModel):
    objects: List[DetectedObject]
    texts: List[DetectedText] = Field(..., description="any alphabetic characters text in the image")

response = interfaze.chat.completions.parse(
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "Get the position of the crane in the image and any text"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://r2public.jigsawstack.com/interfaze/examples/construction.png"
                    },
                },
            ],
        }
    ],
    response_format=DetectionSchema,
)

print(response.choices[0].message.parsed)

print("Object Detection Results:", response.precontext[0].result if response.precontext else None)
```

**LangChain SDK · python**

```python
from langchain_core.messages import HumanMessage
from pydantic import BaseModel, Field
from typing import List

class DetectedObject(BaseModel):
    name: str = Field(..., description="describe the object in the image")
    top_left_x: float
    top_left_y: float
    bottom_right_x: float
    bottom_right_y: float

class DetectedText(BaseModel):
    text: str
    top_left_x: float
    top_left_y: float
    bottom_right_x: float
    bottom_right_y: float

class DetectionSchema(BaseModel):
    objects: List[DetectedObject]
    texts: List[DetectedText] = Field(..., description="any alphabetic characters text in the image")

structured_llm = interfaze.with_structured_output(DetectionSchema, include_raw=True)

response = structured_llm.invoke([
    HumanMessage(
        content=[
            {"type": "text", "text": "Get the position of the crane in the image and any text"},
            {
                "type": "image_url",
                "image_url": {
                    "url": "https://r2public.jigsawstack.com/interfaze/examples/construction.png"
                },
            },
        ]
    )
])

print(response["parsed"])

print("Object Detection Results:", response["raw"].response_metadata.get("precontext"))
```

**Bounding boxes mapped to the image**

![Construction site bounding boxes mapped](https://r2public.jigsawstack.com/interfaze/examples/construction_output.png)

**JSON output**

`object` contains the extracted information defined in the schema. `precontext` contains the raw metadata such as bounding boxes and confidence scores.

```json
{
  "object": {
    "objects": [
      {
        "name": "crane",
        "top_left_x": 630,
        "top_left_y": 139,
        "bottom_right_x": 769,
        "bottom_right_y": 225
      }
    ],
    "texts": [
      {
        "text": "09-12-2020 Sat 21:26:51",
        "top_left_x": 218,
        "top_left_y": 88,
        "bottom_right_x": 623,
        "bottom_right_y": 117
      },
      {
        "text": "WRW tower",
        "top_left_x": 1069,
        "top_left_y": 720,
        "bottom_right_x": 1230,
        "bottom_right_y": 751
      }
    ]
  },
  "response": {
    "id": "interfaze-1775001605439",
    "modelId": "interfaze-beta",
    "body": {
      "id": "interfaze-1775001605439",
      "object": "chat.completion",
      "model": "interfaze-beta",
      "usage": {
        "prompt_tokens": 4903,
        "completion_tokens": 8566,
        "total_tokens": 13469
      },
      "precontext": [
        {
          "name": "object_detection",
          "result": {
            "detected_objects": [
              {
                "bounds": {
                  "top_left": {
                    "x": 630,
                    "y": 139
                  },
                  "top_right": {
                    "x": 769,
                    "y": 139
                  },
                  "bottom_left": {
                    "x": 630,
                    "y": 225
                  },
                  "bottom_right": {
                    "x": 769,
                    "y": 225
                  },
                  "width": 139,
                  "height": 86
                },
                "label": "crane"
              }
            ],
            "gui_elements": [
              {
                "type": "text",
                "bounds": {
                  "top_left": {
                    "x": 1140,
                    "y": 722
                  },
                  "top_right": {
                    "x": 1232,
                    "y": 722
                  },
                  "bottom_left": {
                    "x": 1140,
                    "y": 752
                  },
                  "bottom_right": {
                    "x": 1232,
                    "y": 752
                  },
                  "width": 92,
                  "height": 30
                },
                "interactivity": false,
                "content": "tower"
              }
            ]
          }
        },
        {
          "name": "ocr",
          "result": {
            "extracted_text": "09-12-2020 Sat 21:26:51\nWRW tower",
            "sections": [
              {
                "text": "09-12-2020 Sat 21:26:51\nWRW tower",
                "lines": [
                  {
                    "text": "09-12-2020 Sat 21:26:51",
                    "bounds": {
                      "top_left": {
                        "x": 218,
                        "y": 88
                      },
                      "top_right": {
                        "x": 623,
                        "y": 87
                      },
                      "bottom_right": {
                        "x": 623,
                        "y": 117
                      },
                      "bottom_left": {
                        "x": 218,
                        "y": 118
                      },
                      "width": 405,
                      "height": 30
                    },
                    "average_confidence": 0.99,
                    "words": [
                      {
                        "text": "09-12-2020",
                        "bounds": {
                          "top_left": {
                            "x": 219,
                            "y": 90
                          },
                          "top_right": {
                            "x": 392,
                            "y": 88
                          },
                          "bottom_right": {
                            "x": 391,
                            "y": 117
                          },
                          "bottom_left": {
                            "x": 219,
                            "y": 117
                          },
                          "width": 172.5,
                          "height": 28
                        },
                        "confidence": 0.99
                      },
                      {
                        "text": "Sat",
                        "bounds": {
                          "top_left": {
                            "x": 413,
                            "y": 88
                          },
                          "top_right": {
                            "x": 464,
                            "y": 88
                          },
                          "bottom_right": {
                            "x": 463,
                            "y": 117
                          },
                          "bottom_left": {
                            "x": 413,
                            "y": 117
                          },
                          "width": 50.5,
                          "height": 29
                        },
                        "confidence": 1
                      },
                      {
                        "text": "21:26:51",
                        "bounds": {
                          "top_left": {
                            "x": 483,
                            "y": 88
                          },
                          "top_right": {
                            "x": 622,
                            "y": 87
                          },
                          "bottom_right": {
                            "x": 622,
                            "y": 118
                          },
                          "bottom_left": {
                            "x": 483,
                            "y": 117
                          },
                          "width": 139,
                          "height": 30
                        },
                        "confidence": 0.99
                      }
                    ]
                  },
                  {
                    "text": "WRW tower",
                    "bounds": {
                      "top_left": {
                        "x": 1069,
                        "y": 720
                      },
                      "top_right": {
                        "x": 1230,
                        "y": 721
                      },
                      "bottom_right": {
                        "x": 1230,
                        "y": 751
                      },
                      "bottom_left": {
                        "x": 1069,
                        "y": 750
                      },
                      "width": 161,
                      "height": 30
                    },
                    "average_confidence": 0.99,
                    "words": [
                      {
                        "text": "WRW",
                        "bounds": {
                          "top_left": {
                            "x": 1070,
                            "y": 721
                          },
                          "top_right": {
                            "x": 1120,
                            "y": 721
                          },
                          "bottom_right": {
                            "x": 1120,
                            "y": 750
                          },
                          "bottom_left": {
                            "x": 1070,
                            "y": 751
                          },
                          "width": 50,
                          "height": 29.5
                        },
                        "confidence": 0.99
                      },
                      {
                        "text": "tower",
                        "bounds": {
                          "top_left": {
                            "x": 1145,
                            "y": 722
                          },
                          "top_right": {
                            "x": 1227,
                            "y": 723
                          },
                          "bottom_right": {
                            "x": 1227,
                            "y": 750
                          },
                          "bottom_left": {
                            "x": 1145,
                            "y": 750
                          },
                          "width": 82,
                          "height": 27.5
                        },
                        "confidence": 0.99
                      }
                    ]
                  }
                ]
              }
            ],
            "width": 1400,
            "height": 789
          }
        }
      ]
    }
  },
  "finishReason": "stop",
  "usage": {
    "inputTokens": 4903,
    "outputTokens": 8566,
    "totalTokens": 13469
  }
}
```

## Run object detection task with raw output

Running object detection as a single task with `<task>object_detection</task>` in the system message makes it cheaper and faster with a fixed structured output that's pre-defined.

Learn more about [running a task](https://interfaze.ai/docs/run-tasks).

**Interfaze SDK · typescript**

```typescript
const result = await interfaze.tasks.objectDetection("https://r2public.jigsawstack.com/interfaze/examples/construction.png");

console.log(result);
```

**Vercel AI SDK · typescript**

```typescript
import { generateObject } from "ai";

const { object } = await generateObject({
	model: interfaze("interfaze-beta"),
	output: "no-schema",
	system: "<task>object_detection</task>",
	messages: [
		{
			role: "user",
			content: [
				{ type: "text", text: "Get the position of the crane in the image and any text" },
				{
					type: "image",
					mediaType: "image/png",
					image: "https://r2public.jigsawstack.com/interfaze/examples/construction.png",
				},
			],
		},
	],
});

console.log(object);
```

**LangChain SDK · typescript**

```typescript
import { HumanMessage, SystemMessage } from "@langchain/core/messages";

const response = await interfaze.invoke([
	new SystemMessage("<task>object_detection</task>"),
	new HumanMessage({
		content: [
			{ type: "text", text: "Get the position of the crane in the image and any text" },
			{
				type: "image_url",
				image_url: {
					url: "https://r2public.jigsawstack.com/interfaze/examples/construction.png",
				},
			},
		],
	}),
]);

const { result } = JSON.parse(response.content as string);
console.log(result);
```

**Interfaze SDK · python**

```python
result = interfaze.tasks.object_detection("https://r2public.jigsawstack.com/interfaze/examples/construction.png")

print(result)
```

**LangChain SDK · python**

```python
import json
from langchain_core.messages import SystemMessage, HumanMessage

response = interfaze.invoke([
    SystemMessage(content="<task>object_detection</task>"),
    HumanMessage(content=[
        {"type": "text", "text": "Get the position of the crane in the image and any text"},
        {
            "type": "image_url",
            "image_url": {
                "url": "https://r2public.jigsawstack.com/interfaze/examples/construction.png"
            },
        },
    ]),
])

print(json.loads(response.content).get("result"))
```

**JSON output**

```json
{
  "object": {
    "name": "object_detection",
    "result": {
      "detected_objects": [
        {
          "bounds": {
            "top_left": { "x": 630, "y": 139 },
            "top_right": { "x": 769, "y": 139 },
            "bottom_left": { "x": 630, "y": 225 },
            "bottom_right": { "x": 769, "y": 225 },
            "width": 139,
            "height": 86
          },
          "label": "crane"
        }
      ],
      "gui_elements": [
        {
          "type": "text",
          "bounds": {
            "top_left": { "x": 1140, "y": 722 },
            "top_right": { "x": 1232, "y": 722 },
            "bottom_left": { "x": 1140, "y": 752 },
            "bottom_right": { "x": 1232, "y": 752 },
            "width": 92,
            "height": 30
          },
          "interactivity": false,
          "content": "tower"
        }
      ]
    }
  }
}
```