SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix(ci): benchmarks dashboard was failing due to missing quotations (#34100)

Browse Source
This commit is contained in:
Luc Georges
2024-10-11 19:52:06 +02:00
committed by GitHub
parent 144852fb6b
commit 617b21273a
1 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
benchmark/grafana_dashboard.json
View File

@@ -148,7 +148,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT commit_id as commit_id, commit_message, gpu_name FROM benchmarks WHERE branch = ${branch};", "rawSql": "SELECT commit_id as commit_id, commit_message, gpu_name FROM benchmarks WHERE branch = '${branch}';",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -283,7 +283,7 @@
"id": 7, "id": 7,
"options": { "options": {
"barRadius": 0.05, "barRadius": 0.05,
"barWidth": 0.3, "barWidth": 0.8,
"fullHighlight": false, "fullHighlight": false,
"groupWidth": 0.7, "groupWidth": 0.7,
"legend": { "legend": {
@@ -312,7 +312,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'first_eager_forward_pass_time_secs' AS double precision) AS first_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'first_eager_forward_pass_time_secs' AS double precision) AS first_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -424,7 +424,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'second_eager_forward_pass_time_secs' AS double precision) AS second_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'second_eager_forward_pass_time_secs' AS double precision) AS second_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -545,7 +545,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'time_to_first_token_secs' AS double precision) AS time_to_first_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'time_to_first_token_secs' AS double precision) AS time_to_first_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -653,7 +653,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'time_to_second_token_secs' AS double precision) AS time_to_second_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'time_to_second_token_secs' AS double precision) AS time_to_second_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -761,7 +761,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'time_to_third_token_secs' AS double precision) AS time_to_third_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'time_to_third_token_secs' AS double precision) AS time_to_third_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -869,7 +869,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'time_to_next_token_mean_secs' AS double precision) AS time_to_next_token_mean_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'time_to_next_token_mean_secs' AS double precision) AS time_to_next_token_mean_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -990,7 +990,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'first_compile_generate_time_secs' AS double precision) AS first_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'", "rawSql": "SELECT CAST(m.measurements->'first_compile_generate_time_secs' AS double precision) AS first_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1098,7 +1098,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'second_compile_generate_time_secs' AS double precision) AS second_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}';", "rawSql": "SELECT CAST(m.measurements->'second_compile_generate_time_secs' AS double precision) AS second_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}';",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1206,7 +1206,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'third_compile_generate_time_secs' AS double precision) AS third_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}';", "rawSql": "SELECT CAST(m.measurements->'third_compile_generate_time_secs' AS double precision) AS third_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}';",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1314,7 +1314,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT CAST(m.measurements->'fourth_compile_generate_time_secs' AS double precision) AS fourth_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}';", "rawSql": "SELECT CAST(m.measurements->'fourth_compile_generate_time_secs' AS double precision) AS fourth_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND gpu_name = '${gpu_name}';",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1442,7 +1442,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT\n d.cpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = ${branch}", "rawSql": "SELECT\n d.cpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1627,7 +1627,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT\n b.commit_id,\n d.gpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = ${branch}", "rawSql": "SELECT\n b.commit_id,\n d.gpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1812,7 +1812,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT d.mem_megabytes, d.time FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = ${branch}", "rawSql": "SELECT d.mem_megabytes, d.time FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -1997,7 +1997,7 @@
"editorMode": "code", "editorMode": "code",
"format": "table", "format": "table",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT\n d.gpu_mem_megabytes,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = ${branch}", "rawSql": "SELECT\n d.gpu_mem_megabytes,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}'",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@@ -2099,13 +2099,9 @@
"list": [ "list": [
{ {
"current": { "current": {
"selected": true, "selected": false,
"text": [ "text": "main",
"refactor/benchmarks" "value": "main"
],
"value": [
"refactor/benchmarks"
]
}, },
"datasource": { "datasource": {
"type": "grafana-postgresql-datasource", "type": "grafana-postgresql-datasource",
@@ -2129,21 +2125,21 @@
{ {
"current": { "current": {
"selected": false, "selected": false,
"text": "1728570853117", "text": "1728662868776",
"value": "1728570853117" "value": "1728662868776"
}, },
"datasource": { "datasource": {
"type": "grafana-postgresql-datasource", "type": "grafana-postgresql-datasource",
"uid": "de0dbhs18ho1sc" "uid": "de0dbhs18ho1sc"
}, },
"definition": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = ${branch} ORDER BY benchmark_id ASC LIMIT 1;", "definition": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = '${branch}' ORDER BY benchmark_id ASC LIMIT 1;",
"description": "", "description": "",
"hide": 2, "hide": 2,
"includeAll": false, "includeAll": false,
"multi": false, "multi": false,
"name": "StartTime", "name": "StartTime",
"options": [], "options": [],
"query": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = ${branch} ORDER BY benchmark_id ASC LIMIT 1;", "query": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = '${branch}' ORDER BY benchmark_id ASC LIMIT 1;",
"refresh": 2, "refresh": 2,
"regex": "", "regex": "",
"skipUrlSync": false, "skipUrlSync": false,
@@ -2153,21 +2149,21 @@
{ {
"current": { "current": {
"selected": false, "selected": false,
"text": "1728657828802", "text": "1728663254125",
"value": "1728657828802" "value": "1728663254125"
}, },
"datasource": { "datasource": {
"type": "grafana-postgresql-datasource", "type": "grafana-postgresql-datasource",
"uid": "de0dbhs18ho1sc" "uid": "de0dbhs18ho1sc"
}, },
"definition": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = ${branch} ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;", "definition": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}' ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;",
"description": "", "description": "",
"hide": 2, "hide": 2,
"includeAll": false, "includeAll": false,
"multi": false, "multi": false,
"name": "EndTime", "name": "EndTime",
"options": [], "options": [],
"query": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = ${branch} ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;", "query": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}' ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;",
"refresh": 2, "refresh": 2,
"regex": "", "regex": "",
"skipUrlSync": false, "skipUrlSync": false,
@@ -2180,6 +2176,10 @@
"text": "NVIDIA A10G", "text": "NVIDIA A10G",
"value": "NVIDIA A10G" "value": "NVIDIA A10G"
}, },
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "de0dbhs18ho1sc"
},
"definition": "SELECT DISTINCT gpu_name FROM benchmarks;", "definition": "SELECT DISTINCT gpu_name FROM benchmarks;",
"hide": 0, "hide": 0,
"includeAll": false, "includeAll": false,
@@ -2197,8 +2197,8 @@
] ]
}, },
"time": { "time": {
"from": "2024-10-11T13:10:01.641Z", "from": "now-1h",
"to": "2024-10-11T13:25:21.783Z" "to": "now"
}, },
"timepicker": { "timepicker": {
"hidden": false "hidden": false
@@ -2206,6 +2206,6 @@
"timezone": "browser", "timezone": "browser",
"title": "Transformers benchmarks", "title": "Transformers benchmarks",
"uid": "fdz33iyzln9c0a", "uid": "fdz33iyzln9c0a",
"version": 9, "version": 11,
"weekStart": "" "weekStart": ""
} }