feat: retry upload on jar not found

internal/managed_job/cycle.go

@@ -2,6 +2,7 @@ package managed_job
 
 import (
 	"flink-kube-operator/internal/crd/v1alpha1"
+	"strings"
 	"time"
 
 	"gitea.com/logicamp/lc"
@@ -32,6 +33,8 @@ func (job *ManagedJob) Cycle() {
 
 	// Init job
 	if job.def.Status.JobStatus == "" {
+		if job.def.Status.LastSavepointPath == nil {
+			if job.def.Status.JarId == nil {
 				err := job.upload()
 				if err != nil {
 					job.crd.Patch(job.def.UID, map[string]interface{}{
@@ -41,8 +44,22 @@ func (job *ManagedJob) Cycle() {
 					})
 					return
 				}
-		err = job.run()
+			}
+			for {
+				err := job.run()
 				if err != nil {
+					if strings.ContainsAny(err.Error(), ".jar does not exist") {
+						err := job.upload()
+						if err != nil {
+							job.crd.Patch(job.def.UID, map[string]interface{}{
+								"status": map[string]interface{}{
+									"error": "[upload-error] " + err.Error(),
+								},
+							})
+							return
+						}
+						continue
+					}
 					job.crd.Patch(job.def.UID, map[string]interface{}{
 						"status": map[string]interface{}{
 							"error": "[run-error] " + err.Error(),
@@ -52,6 +69,11 @@ func (job *ManagedJob) Cycle() {
 				}
 				return
 			}
+		} else {
+			job.restore()
+		}
+		return
+	}
 	// job.crd.SetJobStatus(job.def.UID, v1alpha1.FlinkJobStatus{
 	// 	JobStatus: job.def.Status.JobStatus,
 	// })

internal/managed_job/restore.go

@@ -3,6 +3,7 @@ package managed_job
 import (
 	"errors"
 	"flink-kube-operator/internal/crd/v1alpha1"
+	"strings"
 	"time"
 
 	"gitea.com/logicamp/lc"
@@ -18,10 +19,12 @@ func (job *ManagedJob) restore() error {
 	}
 	if job.def.Status.JarId == nil {
 		err := errors.New("missing  jar id")
-		lc.Logger.Error("[managed-job] [run]", zap.Error(err))
+		lc.Logger.Error("[managed-job] [restore]", zap.Error(err))
 		return err
 	}
-	lc.Logger.Debug("[managed-job] [restore] restoring", zap.String("savepointPath", *job.def.Status.LastSavepointPath))
+	lc.Logger.Info("[managed-job] [restore] restoring job", zap.String("name", job.def.GetName()), zap.String("savepointPath", *job.def.Status.LastSavepointPath))
+	var jobId *string
+	for {
 		runJarResp, err := job.client.RunJar(api.RunOpts{
 			JarID:                 *job.def.Status.JarId,
 			AllowNonRestoredState: true,
@@ -29,25 +32,40 @@ func (job *ManagedJob) restore() error {
 			SavepointPath:         *job.def.Status.LastSavepointPath,
 		})
 		if err != nil {
-		lc.Logger.Error("[managed-job] [run]", zap.Error(err))
+			if strings.ContainsAny(err.Error(), ".jar does not exist") {
+				err := job.upload()
+				if err != nil {
+					job.crd.Patch(job.def.UID, map[string]interface{}{
+						"status": map[string]interface{}{
+							"error": "[upload-error] " + err.Error(),
+						},
+					})
+					return nil
+				}
+				continue
+			}
+			lc.Logger.Error("[managed-job] [restore]", zap.Error(err))
 			return err
 		}
+		jobId = &runJarResp.JobId
 		lc.Logger.Debug("[main] after run jar", zap.Any("run-jar-resp", runJarResp))
+		break
+	}
 
 	// job.def.Status.JobId = &runJarResp.JobId
 	// job.def.Status.JobStatus = v1alpha1.JobStatusCreating
 	// job.def.Status.Error = nil
 	job.crd.Patch(job.def.UID, map[string]interface{}{
 		"status": map[string]interface{}{
-			"jobId":                            &runJarResp.JobId,
+			"jobId":                             jobId,
 			"jobStatus":                         v1alpha1.JobStatusCreating,
 			"lifeCycleStatus":                   v1alpha1.LifeCycleStatusRestoring,
-			"lastRestoredSavepointDate":        job.def.Status.LastRestoredSavepointDate,
+			"lastRestoredSavepointDate":         job.def.Status.LastSavepointDate,
 			"restoredCount":                     job.def.Status.RestoredCount + 1,
-			"lastRestoredSavepointRestoreDate": time.Now().Format(time.RFC3339),
+			"lastRestoredSavepointRestoredDate": time.Now().Format(time.RFC3339),
 			"error":                             nil,
 		},
 	})
 
-	return err
+	return nil
 }

internal/managed_job/run.go

@@ -41,6 +41,7 @@ func (job *ManagedJob) run() error {
 		lc.Logger.Error("[managed-job] [run]", zap.Error(err))
 		return err
 	}
+	lc.Logger.Info("[managed-job] [run] starting job", zap.String("name", job.def.GetName()))
 	runJarResp, err := job.client.RunJar(api.RunOpts{
 		JarID:                 *job.def.Status.JarId,
 		AllowNonRestoredState: true,