Created
September 17, 2024 01:53
-
-
Save kmott/7d34abacf46b92ad69193f9619101b3f to your computer and use it in GitHub Desktop.
JuiceFS Nomad Example
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| job "juicefs-controller" { | |
| datacenters = ["kitchen"] | |
| type = "system" | |
| group "juicefs-controller" { | |
| task "juicefs-controller" { | |
| driver = "docker" | |
| config { | |
| image = "juicedata/juicefs-csi-driver:v0.24.7" | |
| args = [ | |
| "--endpoint=unix://csi/csi.sock", | |
| "--logtostderr", | |
| "--nodeid=test", | |
| "--v=5", | |
| "--by-process=true" | |
| ] | |
| privileged = true | |
| } | |
| csi_plugin { | |
| id = "juicefs" | |
| type = "controller" | |
| mount_dir = "/csi" | |
| } | |
| resources { | |
| cpu = 101 | |
| memory = 1024 | |
| } | |
| env { | |
| POD_NAME = "csi-controller" | |
| MINIO_REGION = "us-west" | |
| } | |
| } | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| job "juicefs-node" { | |
| datacenters = ["kitchen"] | |
| type = "system" | |
| group "juicefs-node" { | |
| task "juicefs-node" { | |
| driver = "docker" | |
| config { | |
| image = "juicedata/juicefs-csi-driver:v0.24.7" | |
| args = [ | |
| "--endpoint=unix://csi/csi.sock", | |
| "--logtostderr", | |
| "--v=5", | |
| "--nodeid=test", | |
| "--by-process=true", | |
| ] | |
| privileged = true | |
| } | |
| csi_plugin { | |
| id = "juicefs" | |
| type = "node" | |
| mount_dir = "/csi" | |
| } | |
| resources { | |
| cpu = 1000 | |
| memory = 1024 | |
| } | |
| env { | |
| POD_NAME = "csi-node" | |
| MINIO_REGION = "us-west" | |
| } | |
| } | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| id = "database-juicefs" | |
| name = "database-juicefs" | |
| type = "csi" | |
| plugin_id = "juicefs" | |
| capacity_min = "1G" | |
| capacity_max = "1G" | |
| capability { | |
| access_mode = "multi-node-multi-writer" | |
| attachment_mode = "file-system" | |
| } | |
| mount_options { | |
| fs_type = "ext4" | |
| mount_flags = ["noatime"] | |
| } | |
| secrets { | |
| name="database-juicefs" | |
| metaurl="etcd://root:[email protected]:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org" | |
| bucket="http://minio.nomad.kitchen.example.org:9000/kitchen" | |
| storage="minio" | |
| access-key="administrator" | |
| secret-key="dead-beef" | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| job "database-juicefs" { | |
| datacenters = ["kitchen"] | |
| type = "service" | |
| group "database-juicefs" { | |
| restart { | |
| interval = "10m" | |
| attempts = 20 | |
| delay = "30s" | |
| mode = "fail" | |
| } | |
| ephemeral_disk { | |
| size = 128 | |
| } | |
| network { | |
| mode = "bridge" | |
| } | |
| volume "database-juicefs" { | |
| type = "csi" | |
| source = "database-juicefs" | |
| attachment_mode = "file-system" | |
| access_mode = "single-node-writer" | |
| } | |
| task "database-juicefs" { | |
| driver = "docker" | |
| config { | |
| image = "debian:12" | |
| cmd = [ "tail", "-f", "/dev/null" ] | |
| } | |
| volume_mount { | |
| volume = "database-juicefs" | |
| destination = "/opt/database" | |
| } | |
| resources { | |
| cpu = 100 | |
| memory = 1024 | |
| } | |
| } | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| I0917 01:41:15.835499 8 main.go:121] Run CSI node | |
| I0917 01:41:15.836540 8 driver.go:50] Driver: csi.juicefs.com version v0.24.7-dirty commit ebd4ee6686fab6ec9655b39362a5363454058d3d date 2024-09-04T06:00:07Z | |
| I0917 01:41:16.019738 8 driver.go:115] Listening for connection on address: &net.UnixAddr{Name:"/csi/csi.sock", Net:"unix"} | |
| I0917 01:41:29.375543 8 node.go:107] NodePublishVolume: volume_id is database-juicefs | |
| I0917 01:41:29.375591 8 node.go:118] NodePublishVolume: volume_capability is mount:<fs_type:"ext4" mount_flags:"noatime" > access_mode:<mode:SINGLE_NODE_WRITER > | |
| I0917 01:41:29.375713 8 node.go:124] NodePublishVolume: creating dir /local/csi/per-alloc/43574280-6cea-0311-6996-268521cdc791/database-juicefs/rw-file-system-single-node-writer | |
| I0917 01:41:29.375930 8 node.go:139] NodePublishVolume: volume context: map[capacity:1000000000 subPath:database-juicefs] | |
| I0917 01:41:29.375978 8 node.go:149] NodePublishVolume: mounting juicefs with secret [bucket metaurl name secret-key storage access-key], options [noatime] | |
| W0917 01:41:29.376019 8 juicefs.go:352] Get PV with volumeID database-juicefs error: k8s client is nil | |
| I0917 01:41:29.378418 8 juicefs.go:984] ceFormat cmd: [/usr/local/bin/juicefs format --storage=minio --bucket=http://minio.nomad.kitchen.example.org:9000/kitchen --access-key=administrator --secret-key=${secretkey} ${metaurl} database-juicefs] | |
| I0917 01:41:29.487111 8 juicefs.go:1004] Format output is 2024/09/17 01:41:29.480359 juicefs[19] <INFO>: Meta address: $'etcd://root:****@node1.nomad.kitchen.example.org:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org' [interface.go:504] | |
| 2024/09/17 01:41:29.480516 juicefs[19] <FATAL>: Invalid meta driver: $'etcd [interface.go:507] | |
| I0917 01:41:29.487135 8 juicefs.go:1007] Format error: exit status 1 | |
| E0917 01:41:29.487206 8 driver.go:102] GRPC error: rpc error: code = Internal desc = Could not mount juicefs: juicefs format error: 2024/09/17 01:41:29.480359 juicefs[19] <INFO>: Meta address: $'etcd://root:****@node1.nomad.kitchen.example.org:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org' [interface.go:504] | |
| 2024/09/17 01:41:29.480516 juicefs[19] <FATAL>: Invalid meta driver: $'etcd [interface.go:507] | |
| : exit status 1 | |
| I0917 01:41:30.189715 8 node.go:212] NodeUnpublishVolume: volume_id is database-juicefs | |
| I0917 01:41:30.190200 8 process_mount.go:257] ProcessUmount: /local/csi/per-alloc/43574280-6cea-0311-6996-268521cdc791/database-juicefs/rw-file-system-single-node-writer target not mounted | |
| I0917 01:41:30.190563 8 process_mount.go:302] ProcessUmount: /local/csi/per-alloc/43574280-6cea-0311-6996-268521cdc791/database-juicefs/rw-file-system-single-node-writer target not mounted | |
| I0917 01:42:01.284579 8 node.go:107] NodePublishVolume: volume_id is database-juicefs | |
| I0917 01:42:01.284605 8 node.go:118] NodePublishVolume: volume_capability is mount:<fs_type:"ext4" mount_flags:"noatime" > access_mode:<mode:SINGLE_NODE_WRITER > | |
| I0917 01:42:01.284655 8 node.go:124] NodePublishVolume: creating dir /local/csi/per-alloc/b0f64a60-ff22-a829-72f0-6c8025507b85/database-juicefs/rw-file-system-single-node-writer | |
| I0917 01:42:01.284917 8 node.go:139] NodePublishVolume: volume context: map[capacity:1000000000 subPath:database-juicefs] | |
| I0917 01:42:01.284938 8 node.go:149] NodePublishVolume: mounting juicefs with secret [bucket metaurl name secret-key storage access-key], options [noatime] | |
| W0917 01:42:01.284952 8 juicefs.go:352] Get PV with volumeID database-juicefs error: k8s client is nil | |
| I0917 01:42:01.285226 8 juicefs.go:984] ceFormat cmd: [/usr/local/bin/juicefs format --storage=minio --bucket=http://minio.nomad.kitchen.example.org:9000/kitchen --access-key=administrator --secret-key=${secretkey} ${metaurl} database-juicefs] | |
| I0917 01:42:01.385773 8 juicefs.go:1004] Format output is 2024/09/17 01:42:01.374316 juicefs[26] <INFO>: Meta address: $'etcd://root:****@node1.nomad.kitchen.example.org:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org' [interface.go:504] | |
| 2024/09/17 01:42:01.374730 juicefs[26] <FATAL>: Invalid meta driver: $'etcd [interface.go:507] | |
| I0917 01:42:01.385796 8 juicefs.go:1007] Format error: exit status 1 | |
| E0917 01:42:01.385854 8 driver.go:102] GRPC error: rpc error: code = Internal desc = Could not mount juicefs: juicefs format error: 2024/09/17 01:42:01.374316 juicefs[26] <INFO>: Meta address: $'etcd://root:****@node1.nomad.kitchen.example.org:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org' [interface.go:504] | |
| 2024/09/17 01:42:01.374730 juicefs[26] <FATAL>: Invalid meta driver: $'etcd [interface.go:507] | |
| : exit status 1 | |
| I0917 01:42:02.129180 8 node.go:212] NodeUnpublishVolume: volume_id is database-juicefs | |
| I0917 01:42:02.129530 8 process_mount.go:257] ProcessUmount: /local/csi/per-alloc/b0f64a60-ff22-a829-72f0-6c8025507b85/database-juicefs/rw-file-system-single-node-writer target not mounted | |
| I0917 01:42:02.129817 8 process_mount.go:302] ProcessUmount: /local/csi/per-alloc/b0f64a60-ff22-a829-72f0-6c8025507b85/database-juicefs/rw-file-system-single-node-writer target not mounted | |
| I0917 01:42:32.952950 8 node.go:107] NodePublishVolume: volume_id is database-juicefs | |
| I0917 01:42:32.952970 8 node.go:118] NodePublishVolume: volume_capability is mount:<fs_type:"ext4" mount_flags:"noatime" > access_mode:<mode:SINGLE_NODE_WRITER > | |
| I0917 01:42:32.953003 8 node.go:124] NodePublishVolume: creating dir /local/csi/per-alloc/f9ffab7c-602d-7a5b-2232-c34b3f368ed2/database-juicefs/rw-file-system-single-node-writer | |
| I0917 01:42:32.953170 8 node.go:139] NodePublishVolume: volume context: map[capacity:1000000000 subPath:database-juicefs] | |
| I0917 01:42:32.953189 8 node.go:149] NodePublishVolume: mounting juicefs with secret [metaurl name secret-key storage access-key bucket], options [noatime] | |
| W0917 01:42:32.953208 8 juicefs.go:352] Get PV with volumeID database-juicefs error: k8s client is nil | |
| I0917 01:42:32.953441 8 juicefs.go:984] ceFormat cmd: [/usr/local/bin/juicefs format --storage=minio --bucket=http://minio.nomad.kitchen.example.org:9000/kitchen --access-key=administrator --secret-key=${secretkey} ${metaurl} database-juicefs] | |
| I0917 01:42:33.070526 8 juicefs.go:1004] Format output is 2024/09/17 01:42:33.065408 juicefs[32] <INFO>: Meta address: $'etcd://root:****@node1.nomad.kitchen.example.org:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org' [interface.go:504] | |
| 2024/09/17 01:42:33.065661 juicefs[32] <FATAL>: Invalid meta driver: $'etcd [interface.go:507] | |
| I0917 01:42:33.070557 8 juicefs.go:1007] Format error: exit status 1 | |
| E0917 01:42:33.070598 8 driver.go:102] GRPC error: rpc error: code = Internal desc = Could not mount juicefs: juicefs format error: 2024/09/17 01:42:33.065408 juicefs[32] <INFO>: Meta address: $'etcd://root:****@node1.nomad.kitchen.example.org:2379,node2.nomad.kitchen.example.org:2379,node3.nomad.kitchen.example.org:2379/database-juicefs?insecure-skip-verify=1&server-name=juicefs.kitchen.example.org' [interface.go:504] | |
| 2024/09/17 01:42:33.065661 juicefs[32] <FATAL>: Invalid meta driver: $'etcd [interface.go:507] | |
| : exit status 1 | |
| I0917 01:42:33.912007 8 node.go:212] NodeUnpublishVolume: volume_id is database-juicefs | |
| I0917 01:42:33.912299 8 process_mount.go:257] ProcessUmount: /local/csi/per-alloc/f9ffab7c-602d-7a5b-2232-c34b3f368ed2/database-juicefs/rw-file-system-single-node-writer target not mounted | |
| I0917 01:42:33.912497 8 process_mount.go:302] ProcessUmount: /local/csi/per-alloc/f9ffab7c-602d-7a5b-2232-c34b3f368ed2/database-juicefs/rw-file-system-single-node-writer target not mounted |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment