update readmes

Signed-off-by: Pankaj Walke <[email protected]>

examples/spark/pi.yaml

@@ -0,0 +1,44 @@
+apiVersion: sparkoperator.k8s.io/v1beta2
+kind: SparkApplication
+metadata:
+  name: pyspark-pi-karpenter
+spec:
+  type: Python
+  pythonVersion: "3"
+  mode: cluster
+  image: public.ecr.aws/r1l5w1y9/spark-operator:3.2.1-hadoop-3.3.1-java-11-scala-2.12-python-3.8-latest
+  imagePullPolicy: Always
+  mainApplicationFile: local:///opt/spark/examples/src/main/python/pi.py
+  sparkVersion: 3.1.1
+  restartPolicy:
+    type: OnFailure
+    onFailureRetries: 1
+    onFailureRetryInterval: 10
+    onSubmissionFailureRetries: 5
+    onSubmissionFailureRetryInterval: 20
+  driver:
+    cores: 1
+    coreLimit: 1200m
+    memory: 512m
+    labels:
+      version: 3.1.1
+    serviceAccount: spark
+    nodeSelector:
+      NodeGroupType: spark-spot-ca
+    tolerations:
+      - key: spark-spot-ca
+        operator: Exists
+        effect: NoSchedule
+  executor:
+    cores: 1
+    instances: 2
+    memory: 512m
+    serviceAccount: spark
+    labels:
+      version: 3.1.1
+    nodeSelector:
+      NodeGroupType: spark-spot-ca
+    tolerations:
+      - key: spark-spot-ca
+        operator: Exists
+        effect: NoSchedule

examples/template-generation/data-on-eks.yaml

@@ -0,0 +1,94 @@
+apiVersion: scaffolder.backstage.io/v1beta3
+kind: Template
+metadata:
+  description: Creates a EKS cluster with Spark Operator + extra enabled through Terraform
+  name: data-on-eks-spark
+  title: Data on EKS - Spark Operator
+spec:
+  owner: guest
+  type: service
+  parameters:
+    - title: Terraform config options
+      properties:
+        tfVars: # this field is to be generated.
+          title: Terraform variables
+          type: object
+    - title: Configuration Options
+      properties:
+        name:
+          title: name of this entry
+          type: string
+        namespace:
+          title: namespace within the kubernetes cluster to deploy this
+          type: string
+          default: data-on-eks
+        adminRoleName: # update the aws-config cm to give admin access to this group.
+          title: Admin Role Name
+          description: Name of the role to give the administrative rights on the EKS cluster.
+          default: Admin
+          type: string
+        clusterName:
+          title: Cluster to run
+          description: The cluster to run this workflow in. 
+          type: string
+          ui:field: KubernetesClusterPicker
+        repoUrl: # need a place to store this entity information.
+          title: Repository Location
+          type: string
+          ui:field: RepoUrlPicker
+          ui:options:
+            allowedHosts:
+              - github.com
+  steps:
+    - id: create-repo
+      name: Create Repository
+      action: github:repo:create
+      input:
+        repoUrl: ${{ parameters.repoUrl }}
+    - id: fetch-base
+      name: Fetch Base
+      action: fetch:template
+      input:
+        url: ./data-on-eks
+        values:
+          name: ${{ parameters.name }}
+          namespace: ${{ parameters.namespace }}
+          tfVars: ${{ parameters.tfVars }}
+          adminRoleName: ${{parameters.adminRoleName}}
+          clusterName: ${{ parameters.clusterName }}
+    - id: init-repo
+      name: Initialize Repository
+      action: github:repo:push
+      input:
+        repoUrl: ${{ parameters.repoUrl }}
+        defaultBranch: main
+    - id: wait
+      name: Waiting for the repo to be ready
+      action: "roadiehq:utils:sleep"
+      input:
+        amount: 5
+    - id: register
+      name: Register
+      action: catalog:register
+      input:
+        repoContentsUrl: ${{ steps['init-repo'].output.repoContentsUrl }}
+        catalogInfoPath: '/catalog-info.yaml'
+    - id: apply-cm
+      name: Apply TF variables CM
+      action: cnoe:kubernetes:apply
+      input:
+        manifestPath: cm.yaml # this contains tfvar json file.
+        namespaced: true
+        clusterName: ${{ parameters.clusterName }}
+    - id: apply-workflow
+      name: Start TF workflow
+      action: cnoe:kubernetes:apply
+      input:
+        manifestPath: workflow.yaml # initiates an argo workflows run.
+        namespaced: true
+        clusterName: ${{ parameters.clusterName }}
+  output:
+    links:
+      - title: Open in catalog
+        icon: catalog
+        entityRef: ${{ steps['register'].output.entityRef }}