diff --git a/config.example/dgxie.yml b/config.example/dgxie.yml index c59881044..9653d920e 100644 --- a/config.example/dgxie.yml +++ b/config.example/dgxie.yml @@ -30,9 +30,6 @@ mgmtIntPrv: eth1 # Network domain netDomain: local -# IP address of private network interface on management server -netPrvIp: 192.168.1.1 - # Private Network netPrvNet: 192.168.1.0 @@ -53,6 +50,10 @@ netPrvDhcpEnd: 192.168.1.199 # DHCP lease time netPrvDhcpLease: 7200 +replicaCount: 2 + +pixiecoreSigningKey: 'mysecret' + # HTTPS proxy to use in preseed #httpsProxy: "http://192.168.2.1:3128" diff --git a/containers/dgxie/Dockerfile b/containers/dgxie/Dockerfile index aa9a8d5f6..0bec9ed0d 100644 --- a/containers/dgxie/Dockerfile +++ b/containers/dgxie/Dockerfile @@ -1,13 +1,29 @@ -FROM ubuntu:16.04 +FROM golang:1.10.2 + +ENV COMMIT_HASH 27896bc470a2044c714101b2a321d6d900ccecbe +ENV CUSTOM_FORK_AUTHOR hightoxicity +RUN apt-get update +RUN apt-get install -qy --no-install-recommends wget git +RUN [ -d ${GOPATH}/bin ] || mkdir ${GOPATH}/bin +RUN go get -u github.com/golang/dep/cmd/dep +RUN mkdir -p ${GOPATH}/src/go.universe.tf +WORKDIR /go/src/go.universe.tf +RUN git clone https://github.com/google/netboot.git +WORKDIR /go/src/go.universe.tf/netboot +RUN git remote add ${CUSTOM_FORK_AUTHOR} https://github.com/${CUSTOM_FORK_AUTHOR}/netboot.git && git fetch ${CUSTOM_FORK_AUTHOR} && git checkout ${COMMIT_HASH} +RUN dep ensure +RUN ls -al ./vendor +WORKDIR /go/src +RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/pixiecore -ldflags "-w -s -v -extldflags -static" go.universe.tf/netboot/cmd/pixiecore +FROM ubuntu:16.04 MAINTAINER Douglas Holt RUN apt-get update && \ apt-get -y install apt-transport-https curl && \ - curl -L https://packagecloud.io/danderson/pixiecore/gpgkey | apt-key add - && \ - echo "deb https://packagecloud.io/danderson/pixiecore/debian stretch main" >/etc/apt/sources.list.d/pixiecore.list && \ - apt-get update && \ - apt-get -y install pixiecore nginx vsftpd iptables dnsmasq python-flask + apt-get -y install nginx vsftpd iptables dnsmasq python-flask +COPY --from=0 /bin/pixiecore /usr/bin/pixiecore +RUN chmod +x /usr/bin/pixiecore RUN mkdir -p /www /var/run/vsftpd/empty # && \ diff --git a/containers/dgxie/dnsmasq.conf b/containers/dgxie/dnsmasq.conf index 4fcf33276..fe0247651 100644 --- a/containers/dgxie/dnsmasq.conf +++ b/containers/dgxie/dnsmasq.conf @@ -20,7 +20,7 @@ log-facility=/var/log/dnsmasq.log #enable-tftp #tftp-root=/tftpboot -dhcp-authoritative +#dhcp-authoritative dhcp-range=#DHCP_START#,#DHCP_END#,#LEASETIME# #dhcp-option-force=209,efi64/pxelinux.cfg dhcp-option=tag:green,option:domain-search,#DOMAIN# diff --git a/containers/dgxie/misc/pixiecore b/containers/dgxie/misc/pixiecore deleted file mode 100755 index 82a2416b7..000000000 Binary files a/containers/dgxie/misc/pixiecore and /dev/null differ diff --git a/containers/dgxie/start b/containers/dgxie/start index ebed35b5c..9fde72995 100755 --- a/containers/dgxie/start +++ b/containers/dgxie/start @@ -24,8 +24,43 @@ LEASETIME=${LEASETIME:-7200} DOMAIN=${DOMAIN:-local} HTTPS_PROXY=${HTTPS_PROXY:-} EXTRA_PACKAGES=${EXTRA_PACKAGES:-} +MY_POD_NAME=${MY_POD_NAME:-} +NUM_OF_REPLICAS=${NUM_OF_REPLICAS:-1} ### +iptoi() +{ + a=$(echo "${1}" | cut -d"." -f1) + b=$(echo "${1}" | cut -d"." -f2) + c=$(echo "${1}" | cut -d"." -f3) + d=$(echo "${1}" | cut -d"." -f4) + + echo "$(((a<<24)+(b<<16)+(c<<8)+d))" +} + +itoip() +{ + #returns the dotted-decimal ascii form of an IP arg passed in integer format + echo -n $(($(($(($((${1}/256))/256))/256))%256)). + echo -n $(($(($((${1}/256))/256))%256)). + echo -n $(($((${1}/256))%256)). + echo $((${1}%256)) +} + +SHARD_IDX=$(( ${MY_POD_NAME##*-} )) + +if [ "${NUM_OF_REPLICAS}" -gt 1 ]; then + STARTINT=$(iptoi ${DHCP_START}) + ENDINT=$(iptoi ${DHCP_END}) + + IPCOUNT=$(( ${ENDINT} - ${STARTINT} + 1 )) + + SHARDIPCOUNT=$(( ${IPCOUNT} / ${NUM_OF_REPLICAS} )) + + DHCP_START=$(itoip $(( STARTINT + (SHARD_IDX * SHARDIPCOUNT) ))) + DHCP_END=$(itoip $(( STARTINT + ((SHARD_IDX + 1) * (SHARDIPCOUNT) - 1) ))) +fi + mkdir -p "${ISO}" # Check for mounted DGX Server ISO @@ -100,6 +135,6 @@ nginx & /usr/sbin/vsftpd /etc/vsftpd.conf & /usr/local/bin/rest_api.py >/dev/null 2>&1 & python /api.py & -/usr/bin/pixiecore api http://127.0.0.1:${HTTP_PORT} --dhcp-no-bind -p 81 & +/usr/bin/pixiecore api http://127.0.0.1:${HTTP_PORT} --dhcp-no-bind -p 81 --signing-key ${PIXIECORE_SIGNING_KEY} & dnsmasq tail -f /var/log/dnsmasq.log diff --git a/services/dgxie/templates/deployment.yaml b/services/dgxie/templates/deployment.yaml index d39c3a708..9423b09a5 100644 --- a/services/dgxie/templates/deployment.yaml +++ b/services/dgxie/templates/deployment.yaml @@ -1,5 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: pixiecore +type: Opaque +data: + signingkey: {{ .Values.pixiecoreSigningKey | b64enc | b64enc }} +--- apiVersion: apps/v1beta2 -kind: Deployment +kind: StatefulSet metadata: name: {{ template "dgxie.fullname" . }} labels: @@ -72,7 +80,9 @@ spec: value: "{{ .Values.netDomain }}" # IP address of private network interface on management server - name: IP - value: "{{ .Values.netPrvIp }}" + valueFrom: + fieldRef: + fieldPath: status.podIP # Private Network - name: NETWORK value: "{{ .Values.netPrvNet }}" @@ -102,6 +112,17 @@ spec: value: "{{ .Values.httpsProxy }}" - name: EXTRA_PACKAGES value: "{{ .Values.extraPackages }}" + - name: NUM_OF_REPLICAS + value: "{{ .Values.replicaCount }}" + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PIXIECORE_SIGNING_KEY + valueFrom: + secretKeyRef: + name: pixiecore + key: signingkey resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} @@ -132,21 +153,6 @@ spec: clusterName: rook-ceph path: /iso - name: dhcp-leases - persistentVolumeClaim: - claimName: dhcp-leases ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: dhcp-leases - labels: - app: {{ template "dgxie.name" . }} - annotations: - volume.alpha.kubernetes.io/storage-class: default -spec: - storageClassName: rook-ceph-block - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10M + hostPath: + path: /mnt/dhcp-leases + type: DirectoryOrCreate diff --git a/services/dgxie/values.yaml b/services/dgxie/values.yaml index 5b4c9b33d..c1af4ef13 100644 --- a/services/dgxie/values.yaml +++ b/services/dgxie/values.yaml @@ -13,7 +13,6 @@ dgxKernExtra: '' mgmtIntPub: eth0 mgmtIntPrv: eth1 netDomain: local -netPrvIp: 192.168.1.1 netPrvNet: 192.168.1.0 netPrvNetmask: 255.255.255.0 netPrvGateway: 192.168.1.1 @@ -29,6 +28,8 @@ extraPackages: '' # Deployment config replicaCount: 1 +pixiecoreSigningKey: 'mysecret' + image: repository: deepops/dgxie tag: latest