logicamp/flink-kube-operator
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: remove lc dependency

Browse Source
This commit is contained in:
Mohamad Khani
2024-12-12 23:13:54 +03:30
parent 3912ecef44
commit d3fd04b20c
19 changed files with 182 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/crd/finalizer.go
View File

@@ -1,7 +1,8 @@
package crd
import (
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
"github.com/reactivex/rxgo/v2"
"go.uber.org/zap"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -10,7 +11,7 @@ import (
func (crd Crd) manageFinalizer(jobEventObservable rxgo.Observable) {
for j := range jobEventObservable.Observe() {
jobEvent := j.V.(*FlinkJobCrdEvent)
lc.Logger.Debug("[crd] [manage-finalizer] adding finalizer for", zap.String("name", jobEvent.Job.GetName()))
pkg.Logger.Debug("[crd] [manage-finalizer] adding finalizer for", zap.String("name", jobEvent.Job.GetName()))
controllerutil.AddFinalizer(jobEvent.Job, "")
}
}

11
internal/crd/patch.go
View File

@@ -5,7 +5,8 @@ import (
"encoding/json"
"fmt"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
"go.uber.org/zap"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -30,7 +31,7 @@ func (crd *Crd) Patch(jobUid types.UID, patchData map[string]interface{}) error
metaV1.PatchOptions{},
)
if err != nil {
lc.Logger.Error(
pkg.Logger.Error(
"[crd] [status] error patching custom resource status",
zap.String("namespace", job.GetNamespace()),
zap.Error(err),
@@ -39,13 +40,13 @@ func (crd *Crd) Patch(jobUid types.UID, patchData map[string]interface{}) error
}
newJob, err := convertFromUnstructured(unstructuredJob)
if err != nil {
lc.Logger.Error("[crd] [status] error in structure unstructured patched", zap.Error(err))
pkg.Logger.Error("[crd] [status] error in structure unstructured patched", zap.Error(err))
} else {
lc.Logger.Debug("[crd] [status] patched")
pkg.Logger.Debug("[crd] [status] patched")
}
jobs[jobUid] = newJob
if err != nil {
lc.Logger.Error("[crd] [status] ", zap.Error(err))
pkg.Logger.Error("[crd] [status] ", zap.Error(err))
return err
}
return nil

15
internal/crd/watch.go
View File

@@ -5,7 +5,8 @@ import (
"flink-kube-operator/internal/crd/v1alpha1"
"fmt"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
"github.com/reactivex/rxgo/v2"
"go.uber.org/zap"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -19,25 +20,25 @@ func (crd Crd) watchFlinkJobs() rxgo.Observable {
ch := make(chan rxgo.Item)
go func() {
lc.Logger.Debug("[crd] starting watch")
pkg.Logger.Debug("[crd] starting watch")
watcher, err := crd.client.Watch(context.Background(), metaV1.ListOptions{})
if err != nil {
panic(err)
}
defer watcher.Stop()
for event := range watcher.ResultChan() {
lc.Logger.Debug("[crd] event received", zap.Any("type", event.Type))
pkg.Logger.Debug("[crd] event received", zap.Any("type", event.Type))
unstructuredJob := event.Object.(*unstructured.Unstructured)
unstructuredMap, _, err := unstructured.NestedMap(unstructuredJob.Object)
if err != nil {
lc.Logger.Error("cannot create unstructured map", zap.Error(err))
pkg.Logger.Error("cannot create unstructured map", zap.Error(err))
continue
}
job := &v1alpha1.FlinkJob{}
err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredMap, job)
if err != nil {
lc.Logger.Error("cannot convert unstructured to structured", zap.Error(err))
pkg.Logger.Error("cannot convert unstructured to structured", zap.Error(err))
continue
}
@@ -50,11 +51,11 @@ func (crd Crd) watchFlinkJobs() rxgo.Observable {
switch event.Type {
case watch.Bookmark:
case watch.Modified:
lc.Logger.Info("[crd] [watch] flink job updated")
pkg.Logger.Info("[crd] [watch] flink job updated")
fmt.Printf("FlinkJob updated: %s\n", job.GetName())
crd.repsert(job)
case watch.Added:
lc.Logger.Info("[crd] [watch] new flink job created")
pkg.Logger.Info("[crd] [watch] new flink job created")
crd.repsert(job)
case watch.Deleted:
}

5
internal/jar/jar.go
View File

@@ -7,7 +7,8 @@ import (
"os"
"strings"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
api "github.com/logi-camp/go-flink-client"
gonanoid "github.com/matoous/go-nanoid/v2"
"go.uber.org/zap"
@@ -33,7 +34,7 @@ func (JarFile *JarFile) Upload(flinkClient *api.Client) (fileName string, err er
resp, err := flinkClient.UploadJar(JarFile.filePath)
if err != nil {
lc.Logger.Error("[main] error uploading jar", zap.Error(err))
pkg.Logger.Error("[main] error uploading jar", zap.Error(err))
}
filePathParts := strings.Split(resp.FileName, "/")
fileName = filePathParts[len(filePathParts)-1]

7
internal/managed_job/cycle.go
View File

@@ -5,7 +5,8 @@ import (
"strings"
"time"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
"go.uber.org/zap"
)
@@ -29,7 +30,7 @@ import (
// }
func (job *ManagedJob) Cycle() {
lc.Logger.Debug("[managed-job] [new] check cycle", zap.String("jobKey", string(job.def.UID)))
pkg.Logger.Debug("[managed-job] [new] check cycle", zap.String("jobKey", string(job.def.UID)))
// Init job
if job.def.Status.JobStatus == "" {
@@ -102,5 +103,5 @@ func (job *ManagedJob) Cycle() {
return
}
lc.Logger.Warn("[managed-job] [cycle]", zap.String("unhanded job status", string(job.def.Status.JobStatus)))
pkg.Logger.Warn("[managed-job] [cycle]", zap.String("unhanded job status", string(job.def.Status.JobStatus)))
}

13
internal/managed_job/restore.go
View File

@@ -6,7 +6,8 @@ import (
"strings"
"time"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
api "github.com/logi-camp/go-flink-client"
"go.uber.org/zap"
)
@@ -14,15 +15,15 @@ import (
// restore the job from savepoint and jarId in managedJob
func (job *ManagedJob) restore() error {
if job.def.Status.LastSavepointPath == nil {
lc.Logger.Error("[managed-job] [restore]", zap.Error(v1alpha1.ErrNoSavepointPath))
pkg.Logger.Error("[managed-job] [restore]", zap.Error(v1alpha1.ErrNoSavepointPath))
return v1alpha1.ErrNoSavepointPath
}
if job.def.Status.JarId == nil {
err := errors.New("missing jar id")
lc.Logger.Error("[managed-job] [restore]", zap.Error(err))
pkg.Logger.Error("[managed-job] [restore]", zap.Error(err))
return err
}
lc.Logger.Info("[managed-job] [restore] restoring job", zap.String("name", job.def.GetName()), zap.String("savepointPath", *job.def.Status.LastSavepointPath))
pkg.Logger.Info("[managed-job] [restore] restoring job", zap.String("name", job.def.GetName()), zap.String("savepointPath", *job.def.Status.LastSavepointPath))
var jobId *string
for {
runJarResp, err := job.client.RunJar(api.RunOpts{
@@ -44,11 +45,11 @@ func (job *ManagedJob) restore() error {
}
continue
}
lc.Logger.Error("[managed-job] [restore]", zap.Error(err))
pkg.Logger.Error("[managed-job] [restore]", zap.Error(err))
return err
}
jobId = &runJarResp.JobId
lc.Logger.Debug("[main] after run jar", zap.Any("run-jar-resp", runJarResp))
pkg.Logger.Debug("[main] after run jar", zap.Any("run-jar-resp", runJarResp))
break
}

19
internal/managed_job/run.go
View File

@@ -5,7 +5,8 @@ import (
"flink-kube-operator/internal/crd/v1alpha1"
"flink-kube-operator/internal/jar"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
api "github.com/logi-camp/go-flink-client"
"go.uber.org/zap"
)
@@ -14,19 +15,19 @@ import (
func (job *ManagedJob) upload() error {
jarFile, err := jar.NewJarFile(job.def.Spec.JarURI)
if err != nil {
lc.Logger.Debug("[main] error on download jar", zap.Error(err))
pkg.Logger.Debug("[main] error on download jar", zap.Error(err))
return err
}
fileName, err := jarFile.Upload(job.client)
if err != nil {
lc.Logger.Debug("[main] error on upload jar", zap.Error(err))
pkg.Logger.Debug("[main] error on upload jar", zap.Error(err))
return err
}
err = jarFile.Delete()
if err != nil {
lc.Logger.Debug("[main] error on delete jar", zap.Error(err))
pkg.Logger.Debug("[main] error on delete jar", zap.Error(err))
}
lc.Logger.Debug("[main] after upload jar", zap.Any("upload-jar-resp", fileName))
pkg.Logger.Debug("[main] after upload jar", zap.Any("upload-jar-resp", fileName))
job.def.Status.JarId = &fileName
job.crd.Patch(job.def.UID, map[string]interface{}{
@@ -41,20 +42,20 @@ func (job *ManagedJob) upload() error {
func (job *ManagedJob) run() error {
if job.def.Status.JarId == nil {
err := errors.New("missing jar id")
lc.Logger.Error("[managed-job] [run]", zap.Error(err))
pkg.Logger.Error("[managed-job] [run]", zap.Error(err))
return err
}
lc.Logger.Info("[managed-job] [run] starting job", zap.String("name", job.def.GetName()))
pkg.Logger.Info("[managed-job] [run] starting job", zap.String("name", job.def.GetName()))
runJarResp, err := job.client.RunJar(api.RunOpts{
JarID: *job.def.Status.JarId,
AllowNonRestoredState: true,
EntryClass: job.def.Spec.EntryClass,
})
if err != nil {
lc.Logger.Error("[managed-job] [run]", zap.Error(err))
pkg.Logger.Error("[managed-job] [run]", zap.Error(err))
return err
}
lc.Logger.Debug("[main] after run jar", zap.Any("run-jar-resp", runJarResp))
pkg.Logger.Debug("[main] after run jar", zap.Any("run-jar-resp", runJarResp))
// if job.state == nil {
// job.state = &jobState{}

17
internal/managed_job/savepoint.go
View File

@@ -5,23 +5,24 @@ import (
"strings"
"time"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
api "github.com/logi-camp/go-flink-client"
"go.uber.org/zap"
)
func (job ManagedJob) createSavepoint() error {
if job.def.Status.JobId == nil {
lc.Logger.Debug("[managed-job] [savepoint] no job id")
pkg.Logger.Debug("[managed-job] [savepoint] no job id")
return v1alpha1.ErrNoJobId
}
lc.Logger.Info("[managed-job] [savepoint] creating savepoint", zap.String("interval", job.def.Spec.SavepointInterval.String()))
pkg.Logger.Info("[managed-job] [savepoint] creating savepoint", zap.String("interval", job.def.Spec.SavepointInterval.String()))
resp, err := job.client.SavePoints(*job.def.Status.JobId, "/flink-data/savepoints-2/", false)
if err != nil {
lc.Logger.Error("[managed-job] [savepoint] error in creating savepoint", zap.Error(err))
pkg.Logger.Error("[managed-job] [savepoint] error in creating savepoint", zap.Error(err))
return err
}
lc.Logger.Debug("[managed-job] [savepoint]", zap.Any("savepoint-resp", resp))
pkg.Logger.Debug("[managed-job] [savepoint]", zap.Any("savepoint-resp", resp))
job.crd.Patch(job.def.UID, map[string]interface{}{
"status": map[string]interface{}{
@@ -33,15 +34,15 @@ func (job ManagedJob) createSavepoint() error {
func (job ManagedJob) trackSavepoint() error {
if job.def.Status.JobId == nil {
lc.Logger.Debug("[managed-job] [savepoint] no job id")
pkg.Logger.Debug("[managed-job] [savepoint] no job id")
return v1alpha1.ErrNoJobId
}
if job.def.Status.SavepointTriggerId == nil {
lc.Logger.Debug("[managed-job] [savepoint] no job id")
pkg.Logger.Debug("[managed-job] [savepoint] no job id")
return v1alpha1.ErrNoSavepointTriggerId
}
resp, err := job.client.TrackSavepoint(*job.def.Status.JobId, *job.def.Status.SavepointTriggerId)
lc.Logger.Debug("[managed-job] [savepoint] track savepoint", zap.Any("status.Id", resp.Status.Id), zap.Any("failureCause.stacktrace", resp.Operation.FailureCause.StackTrace), zap.Error(err))
pkg.Logger.Debug("[managed-job] [savepoint] track savepoint", zap.Any("status.Id", resp.Status.Id), zap.Any("failureCause.stacktrace", resp.Operation.FailureCause.StackTrace), zap.Error(err))
if err != nil {
if strings.IndexAny(err.Error(), "http status not 2xx: 404") == 0 {
job.crd.Patch(job.def.UID, map[string]interface{}{

7
internal/managed_job/status.go
View File

@@ -4,18 +4,19 @@ import (
"flink-kube-operator/internal/crd/v1alpha1"
"strings"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
"go.uber.org/zap"
)
func (job *ManagedJob) checkStatus() error {
if job.def.Status.JobId == nil {
lc.Logger.Debug("[managed-job] [status] no job id")
pkg.Logger.Debug("[managed-job] [status] no job id")
return v1alpha1.ErrNoJobId
}
statusResp, err := job.client.Job(*job.def.Status.JobId)
if err != nil {
lc.Logger.Debug("[managed-job] [status] cannot fetch status", zap.Error(err))
pkg.Logger.Debug("[managed-job] [status] cannot fetch status", zap.Error(err))
if strings.IndexAny(err.Error(), "http status not 2xx: 404") == 0 {
// job.updateState(jobState{
// JobId: job.state.JobId,

9
internal/manager/manager.go
View File

@@ -6,7 +6,8 @@ import (
"flink-kube-operator/internal/managed_job"
"time"
"gitea.com/logicamp/lc"
"flink-kube-operator/pkg"
api "github.com/logi-camp/go-flink-client"
"github.com/samber/lo"
"go.uber.org/zap"
@@ -43,7 +44,7 @@ func NewManager(client *api.Client, crdInstance *crd.Crd) Manager {
func (mgr *Manager) cycle(client *api.Client, crdInstance *crd.Crd) {
jobManagerJobOverviews, jobManagerJobStatusError := mgr.client.JobsOverview()
if jobManagerJobStatusError != nil {
lc.Logger.Error("[manager] [cycle] cannot check flink jobs status", zap.Error(jobManagerJobStatusError))
pkg.Logger.Error("[manager] [cycle] cannot check flink jobs status", zap.Error(jobManagerJobStatusError))
crdInstance.PatchAll(map[string]interface{}{
"status": map[string]interface{}{
"jobStatus": "",
@@ -51,7 +52,7 @@ func (mgr *Manager) cycle(client *api.Client, crdInstance *crd.Crd) {
},
})
}
//lc.Logger.Debug("[manager] [cycle] overviews", zap.Any("overviews", jobsOverviews))
//pkg.Logger.Debug("[manager] [cycle] overviews", zap.Any("overviews", jobsOverviews))
// Loop over job definitions as Kubernetes CRD
for _, uid := range crd.GetAllJobKeys() {
@@ -78,7 +79,7 @@ func (mgr *Manager) cycle(client *api.Client, crdInstance *crd.Crd) {
return false
})
if ok {
lc.Logger.Debug("[manager] read status from flink", zap.String("name", jobManagerJobOverview.Name), zap.String("state", jobManagerJobOverview.State))
pkg.Logger.Debug("[manager] read status from flink", zap.String("name", jobManagerJobOverview.Name), zap.String("state", jobManagerJobOverview.State))
var jobLifeCycleStatus *string
if jobManagerJobOverview.State == string(v1alpha1.JobStatusRunning) {
status := string(v1alpha1.LifeCycleStatusHealthy)