{"ok":true,"data":{"publisher":"LLM Tier","source":"https://www.llm-tier.com","licence":"CC-BY-4.0","licence_url":"https://creativecommons.org/licenses/by/4.0/","attribution":"Data by LLM Tier, https://www.llm-tier.com, licensed CC-BY-4.0","generated_at":"2026-08-19T23:48:26.084Z","spec":{"name":"vibeOps","runs_per_tool":5,"prompts_per_run":9,"methodology_url":"https://www.llm-tier.com/methodology"},"filter":{"tool":"devin"},"count":45,"runs":[{"tool_slug":"devin","run_number":1,"prompt_index":1,"started_at":"2026-08-11T08:00:00.000Z","duration_seconds":119.8,"outcome":"fail","notes":"Build did not start after deploy. Recorded as a fail with the log preserved."},{"tool_slug":"devin","run_number":1,"prompt_index":2,"started_at":"2026-08-11T08:03:59.800Z","duration_seconds":294.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":1,"prompt_index":3,"started_at":"2026-08-11T08:10:54.100Z","duration_seconds":661.9,"outcome":"partial","notes":"Required one clarifying instruction before the criterion was met."},{"tool_slug":"devin","run_number":1,"prompt_index":4,"started_at":"2026-08-11T08:23:56.000Z","duration_seconds":931.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":1,"prompt_index":5,"started_at":"2026-08-11T08:41:27.300Z","duration_seconds":352.7,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":1,"prompt_index":6,"started_at":"2026-08-11T08:49:20.000Z","duration_seconds":507.8,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":1,"prompt_index":7,"started_at":"2026-08-11T08:59:47.800Z","duration_seconds":547.5,"outcome":"pass","notes":"Opened a pull request with passing tests and no human in the loop for the whole task."},{"tool_slug":"devin","run_number":1,"prompt_index":8,"started_at":"2026-08-11T09:10:55.300Z","duration_seconds":349.6,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":1,"prompt_index":9,"started_at":"2026-08-11T09:18:44.900Z","duration_seconds":496.3,"outcome":"fail","notes":"Build did not start after deploy. Recorded as a fail with the log preserved."},{"tool_slug":"devin","run_number":2,"prompt_index":1,"started_at":"2026-08-12T08:00:00.000Z","duration_seconds":304.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":2,"prompt_index":2,"started_at":"2026-08-12T08:07:04.300Z","duration_seconds":226.4,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":2,"prompt_index":3,"started_at":"2026-08-12T08:12:50.700Z","duration_seconds":390.6,"outcome":"fail","notes":"Generated migration conflicted with the existing schema and was not reconciled."},{"tool_slug":"devin","run_number":2,"prompt_index":4,"started_at":"2026-08-12T08:21:21.300Z","duration_seconds":462.7,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":2,"prompt_index":5,"started_at":"2026-08-12T08:31:04.000Z","duration_seconds":110.9,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":2,"prompt_index":6,"started_at":"2026-08-12T08:34:54.900Z","duration_seconds":241.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":2,"prompt_index":7,"started_at":"2026-08-12T08:40:56.200Z","duration_seconds":353.6,"outcome":"pass","notes":"Opened a pull request with passing tests and no human in the loop for the whole task."},{"tool_slug":"devin","run_number":2,"prompt_index":8,"started_at":"2026-08-12T08:48:49.800Z","duration_seconds":655.7,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":2,"prompt_index":9,"started_at":"2026-08-12T09:01:45.500Z","duration_seconds":356,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":3,"prompt_index":1,"started_at":"2026-08-13T08:00:00.000Z","duration_seconds":479.9,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":3,"prompt_index":2,"started_at":"2026-08-13T08:09:59.900Z","duration_seconds":241.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":3,"prompt_index":3,"started_at":"2026-08-13T08:16:01.200Z","duration_seconds":364.9,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":3,"prompt_index":4,"started_at":"2026-08-13T08:24:06.100Z","duration_seconds":1293.4,"outcome":"partial","notes":"Error envelope inconsistent between two of the endpoints."},{"tool_slug":"devin","run_number":3,"prompt_index":5,"started_at":"2026-08-13T08:47:39.500Z","duration_seconds":278.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":3,"prompt_index":6,"started_at":"2026-08-13T08:54:17.800Z","duration_seconds":644.9,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":3,"prompt_index":7,"started_at":"2026-08-13T09:07:02.700Z","duration_seconds":350.5,"outcome":"pass","notes":"Opened a pull request with passing tests and no human in the loop for the whole task."},{"tool_slug":"devin","run_number":3,"prompt_index":8,"started_at":"2026-08-13T09:14:53.200Z","duration_seconds":372.8,"outcome":"partial","notes":"Required one clarifying instruction before the criterion was met."},{"tool_slug":"devin","run_number":3,"prompt_index":9,"started_at":"2026-08-13T09:23:06.000Z","duration_seconds":301.8,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":4,"prompt_index":1,"started_at":"2026-08-14T08:00:00.000Z","duration_seconds":151.9,"outcome":"partial","notes":"Validation added on the write path only. Read path left unguarded."},{"tool_slug":"devin","run_number":4,"prompt_index":2,"started_at":"2026-08-14T08:04:31.900Z","duration_seconds":338.1,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":4,"prompt_index":3,"started_at":"2026-08-14T08:12:10.000Z","duration_seconds":235.8,"outcome":"fail","notes":"Generated migration conflicted with the existing schema and was not reconciled."},{"tool_slug":"devin","run_number":4,"prompt_index":4,"started_at":"2026-08-14T08:18:05.800Z","duration_seconds":238,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":4,"prompt_index":5,"started_at":"2026-08-14T08:24:03.800Z","duration_seconds":377,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":4,"prompt_index":6,"started_at":"2026-08-14T08:32:20.800Z","duration_seconds":606.5,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":4,"prompt_index":7,"started_at":"2026-08-14T08:44:27.300Z","duration_seconds":514.1,"outcome":"partial","notes":"Required one clarifying instruction before the criterion was met."},{"tool_slug":"devin","run_number":4,"prompt_index":8,"started_at":"2026-08-14T08:55:01.400Z","duration_seconds":289.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":4,"prompt_index":9,"started_at":"2026-08-14T09:01:50.700Z","duration_seconds":480.2,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":5,"prompt_index":1,"started_at":"2026-08-15T08:00:00.000Z","duration_seconds":359.7,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":5,"prompt_index":2,"started_at":"2026-08-15T08:07:59.700Z","duration_seconds":263.8,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":5,"prompt_index":3,"started_at":"2026-08-15T08:14:23.500Z","duration_seconds":246.2,"outcome":"partial","notes":"Delivered the main path, missed the boundary case named in the pass criteria."},{"tool_slug":"devin","run_number":5,"prompt_index":4,"started_at":"2026-08-15T08:20:29.700Z","duration_seconds":839.6,"outcome":"fail","notes":"Task abandoned by the agent with no artifact produced."},{"tool_slug":"devin","run_number":5,"prompt_index":5,"started_at":"2026-08-15T08:36:29.300Z","duration_seconds":437.7,"outcome":"fail","notes":"Generated migration conflicted with the existing schema and was not reconciled."},{"tool_slug":"devin","run_number":5,"prompt_index":6,"started_at":"2026-08-15T08:45:47.000Z","duration_seconds":409.9,"outcome":"partial","notes":"Required one clarifying instruction before the criterion was met."},{"tool_slug":"devin","run_number":5,"prompt_index":7,"started_at":"2026-08-15T08:54:36.900Z","duration_seconds":321.4,"outcome":"pass","notes":"Opened a pull request with passing tests and no human in the loop for the whole task."},{"tool_slug":"devin","run_number":5,"prompt_index":8,"started_at":"2026-08-15T09:01:58.300Z","duration_seconds":208.3,"outcome":"pass","notes":null},{"tool_slug":"devin","run_number":5,"prompt_index":9,"started_at":"2026-08-15T09:07:26.600Z","duration_seconds":250.9,"outcome":"pass","notes":null}]}}