From 3c6a32c04bf9fa1dcc10d92e0b8b2ecb87347581 Mon Sep 17 00:00:00 2001 From: dongming Date: Sun, 8 Jan 2023 14:31:40 +0800 Subject: [PATCH] =?UTF-8?q?l-101=20=E6=B7=BB=E5=8A=A0e2e=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps.mashibing.com_msbdeployments.yaml | 3 + config/rbac/role.yaml | 22 ++++ controllers/generate-cr.go | 81 +++++++++++- controllers/msbdeployment_controller.go | 50 +++++++- controllers/templates/ingress-with-tls.yaml | 22 ++++ test/e2e/config.yaml | 2 +- test/e2e/create/create-ingress.go | 107 +++++++++++++++- .../testdata/create-ingress-with-tls.yaml | 12 ++ test/e2e/create_test.go | 1 + test/e2e/e2e.xml | 118 ++++++++++-------- 10 files changed, 359 insertions(+), 59 deletions(-) create mode 100644 controllers/templates/ingress-with-tls.yaml create mode 100644 test/e2e/create/testdata/create-ingress-with-tls.yaml diff --git a/config/crd/bases/apps.mashibing.com_msbdeployments.yaml b/config/crd/bases/apps.mashibing.com_msbdeployments.yaml index 685f7dc..3b02844 100644 --- a/config/crd/bases/apps.mashibing.com_msbdeployments.yaml +++ b/config/crd/bases/apps.mashibing.com_msbdeployments.yaml @@ -166,6 +166,9 @@ spec: description: ServicePort service的端口，一般是随机生成，这里我们为了防止冲突，使用和我们提供服务相同的端口。 format: int32 type: integer + tls: + description: Tls 是否开启https + type: boolean required: - mode type: object diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6c81932..4891df6 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -55,6 +55,28 @@ rules: - get - patch - update +- apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - issuers + verbs: + - create + - get + - list + - patch + - update + - watch - apiGroups: - networking.k8s.io resources: diff --git a/controllers/generate-cr.go b/controllers/generate-cr.go index 8ad4725..44b7058 100644 --- a/controllers/generate-cr.go +++ b/controllers/generate-cr.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "text/template" networkv1 "k8s.io/api/networking/v1" @@ -38,7 +39,16 @@ func NewDeployment(md *myAppsv1.MsbDeployment) (*appsv1.Deployment, error) { } func NewIngress(md *myAppsv1.MsbDeployment) (*networkv1.Ingress, error) { - content, err := parseTemplate(md, "ingress.yaml") + var ( + content []byte + err error + ) + if md.Spec.Expose.Tls { + // 添加 tls 的支持 + content, err = parseTemplate(md, "ingress-with-tls.yaml") + } else { + content, err = parseTemplate(md, "ingress.yaml") + } if err != nil { return nil, err } @@ -70,3 +80,72 @@ func NewServiceNP(md *myAppsv1.MsbDeployment) (*corev1.Service, error) { } return svc, nil } + +// NewIssuer 实现创建issuer资源对象 +func NewIssuer(md *myAppsv1.MsbDeployment) (*unstructured.Unstructured, error) { + if md.Spec.Expose.Mode != myAppsv1.ModeIngress || + !md.Spec.Expose.Tls { + return nil, nil + } + // Sample + //apiVersion: cert-manager.io/v1 + //kind: Issuer + //metadata: + // name: selfsigned-issuer + //spec: + // selfSigned: {} + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "cert-manager.io/v1", + "kind": "Issuer", + "metadata": map[string]interface{}{ + "name": md.Name, + "namespace": md.Namespace, + }, + "spec": map[string]interface{}{ + "selfSigned": map[string]interface{}{}, + }, + }, + }, nil +} + +// NewCert 实现创建certificate资源 +func NewCert(md *myAppsv1.MsbDeployment) (*unstructured.Unstructured, error) { + if md.Spec.Expose.Mode != myAppsv1.ModeIngress || + !md.Spec.Expose.Tls { + return nil, nil + } + // Sample + //apiVersion: cert-manager.io/v1 + //kind: Certificate + //metadata: + // name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + // namespace: system + //spec: + // dnsNames: + // - + // issuerRef: + // kind: Issuer + // name: selfsigned-issuer + // secretName: webhook-server-cert + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "cert-manager.io/v1", + "kind": "Certificate", + "metadata": map[string]interface{}{ + "name": md.Name, + "namespace": md.Namespace, + }, + "spec": map[string]interface{}{ + "dnsNames": []interface{}{ + md.Spec.Expose.IngressDomain, + }, + "issuerRef": map[string]interface{}{ + "kind": "Issuer", + "name": md.Name, + }, + "secretName": md.Name, + }, + }, + }, nil +} diff --git a/controllers/msbdeployment_controller.go b/controllers/msbdeployment_controller.go index 23518cb..3f826e0 100644 --- a/controllers/msbdeployment_controller.go +++ b/controllers/msbdeployment_controller.go @@ -333,7 +333,7 @@ func (r *MsbDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&myAppsv1.MsbDeployment{}). Owns(&appsv1.Deployment{}). // 监控 deployment 类型，变更就触发 reconciler - Owns(&corev1.Service{}). // 监控 service 类型，变更就触发 reconciler + Owns(&corev1.Service{}). // 监控 service 类型，变更就触发 reconciler Owns(&networkv1.Ingress{}). // 监控 ingress 类型，变更就触发 reconciler Complete(r) } @@ -587,11 +587,55 @@ func (r *MsbDeploymentReconciler) deleteStatus(md *myAppsv1.MsbDeployment, condi } } -func (r *MsbDeploymentReconciler) createIssuer(ctx context.Context, mdCopy *myAppsv1.MsbDeployment) error { +func (r *MsbDeploymentReconciler) createIssuer(ctx context.Context, md *myAppsv1.MsbDeployment) error { + // 1. 创建 issuer 资源 + i, err := NewIssuer(md) + if err != nil { + return err + } + + // 设置 issuer 所属于 md + if err := controllerutil.SetControllerReference(md, i, r.Scheme); err != nil { + return err + } + + // 在k8s中创建issuer资源 + if _, err := r.DynamicClient.Resource(issuerGVR). + Namespace(md.Namespace). + Create(ctx, i, metav1.CreateOptions{}); err != nil { + if errors.IsAlreadyExists(err) { + // 这是一个折中的考虑，在没有比较完整的处理证书更新的方案前， + // 这是一个简单并且不会出现意外错误的处理方式 + return nil + } + return err + } return nil } -func (r *MsbDeploymentReconciler) createCert(ctx context.Context, mdCopy *myAppsv1.MsbDeployment) error { +func (r *MsbDeploymentReconciler) createCert(ctx context.Context, md *myAppsv1.MsbDeployment) error { + // 1. 创建 issuer 资源 + c, err := NewCert(md) + if err != nil { + return err + } + + // 设置 issuer 所属于 md + if err := controllerutil.SetControllerReference(md, c, r.Scheme); err != nil { + return err + } + + // 在k8s中创建certificate资源 + if _, err := r.DynamicClient.Resource(certGVR). + Namespace(md.Namespace). + Create(ctx, c, metav1.CreateOptions{}); err != nil { + if errors.IsAlreadyExists(err) { + // 这是一个折中的考虑，在没有比较完整的处理证书更新的方案前， + // 这是一个简单并且不会出现意外错误的处理方式 + return nil + } + return err + } return nil } diff --git a/controllers/templates/ingress-with-tls.yaml b/controllers/templates/ingress-with-tls.yaml new file mode 100644 index 0000000..30ab9e3 --- /dev/null +++ b/controllers/templates/ingress-with-tls.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .ObjectMeta.Name }} + namespace: {{ .ObjectMeta.Namespace }} +spec: + ingressClassName: nginx + tls: + - hosts: + - {{ .Spec.Expose.IngressDomain }} + secretName: {{ .ObjectMeta.Name }} + rules: + - host: {{ .Spec.Expose.IngressDomain }} + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: {{ .ObjectMeta.Name }} + port: + number: {{ .Spec.Expose.ServicePort }} \ No newline at end of file diff --git a/test/e2e/config.yaml b/test/e2e/config.yaml index 37a2b91..c1352b5 100644 --- a/test/e2e/config.yaml +++ b/test/e2e/config.yaml @@ -1,7 +1,7 @@ cluster: kind: name: e2e - retain: false + retain: true config: | kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 diff --git a/test/e2e/create/create-ingress.go b/test/e2e/create/create-ingress.go index cbc1585..930648c 100644 --- a/test/e2e/create/create-ingress.go +++ b/test/e2e/create/create-ingress.go @@ -2,6 +2,9 @@ package create import ( "context" + networkv1 "k8s.io/api/networking/v1" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -9,7 +12,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - "time" "mashibing.com/pkg/mashibing-deployment/test/framework" ) @@ -95,6 +97,109 @@ func CreateIngressMsbDeployment(ctx *framework.TestContext, f *framework.Framewo }) } +// 测试HTTPS +func CreateIngressMsbDeploymentWithTls(ctx *framework.TestContext, f *framework.Framework) { + var ( + // 1. 准备测试数据 + ctFilePath = "create/testdata/create-ingress-with-tls.yaml" + obj = &unstructured.Unstructured{Object: make(map[string]interface{})} + dc dynamic.Interface + cs *kubernetes.Clientset + + // 3. 准备测试用到的全局变量 + msbGVR = schema.GroupVersionResource{ + Group: "apps.mashibing.com", + Version: "v1", + Resource: "msbdeployments", + } + // issuer + issuerGVR = schema.GroupVersionResource{ + Group: "cert-manager.io", + Version: "v1", + Resource: "issuers", + } + // certificate + certGVR = schema.GroupVersionResource{ + Group: "cert-manager.io", + Version: "v1", + Resource: "certificates", + } + + err error + ) + BeforeEach(func() { + // 2. 加载测试数据 + err = f.LoadYamlToUnstructured(ctFilePath, obj) + Expect(err).Should(BeNil()) + + // 4. 初始化测试用到的全局变量 + dc = ctx.CreateDynamicClient() + cs = ctx.CreateClientSet() + }) + Context("Create msbdeployment mod ingress with tls", func() { + It("Should be create mod ingress with tls success", func() { + _, err = dc.Resource(msbGVR).Namespace("default").Create(context.TODO(), obj, metav1.CreateOptions{}) + Expect(err).Should(BeNil()) + + By("Sleep 3 second wait creating done") + time.Sleep(3 * time.Second) + }) + It("Should be exist msbdeployment", func() { + _, err = dc.Resource(msbGVR).Namespace("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).Should(BeNil()) + }) + It("Should be exist ingress, and have a tls setting", func() { + var ig *networkv1.Ingress + ig, err = cs.NetworkingV1().Ingresses("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).Should(BeNil()) + Expect(len(ig.Spec.TLS)).To(Equal(1)) + }) + It("Should be exist issuer", func() { + _, err = dc.Resource(issuerGVR).Namespace("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).Should(BeNil()) + }) + It("Should be exist certificate", func() { + _, err = dc.Resource(certGVR).Namespace("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).Should(BeNil()) + }) + }) + + Context("Delete msbdeployment mod ingress with tls", func() { + It("Should be delete mod ingress success", func() { + err = dc.Resource(msbGVR).Namespace("default").Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) + Expect(err).Should(BeNil()) + + By("Sleep 3 second wait deleting done") + time.Sleep(3 * time.Second) + }) + It("Should not be exist msbdeployment", func() { + _, err = dc.Resource(msbGVR).Namespace("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).ShouldNot(BeNil()) + }) + It("Should not be exist deployment", func() { + _, err = cs.AppsV1().Deployments("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).ShouldNot(BeNil()) + }) + It("Should not be exist service", func() { + _, err = cs.CoreV1().Services("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).ShouldNot(BeNil()) + }) + It("Should not be exist ingress", func() { + _, err = cs.NetworkingV1().Ingresses("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).ShouldNot(BeNil()) + }) + It("Should not be exist issuer", func() { + _, err = dc.Resource(issuerGVR).Namespace("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).ShouldNot(BeNil()) + }) + It("Should not be exist certificate", func() { + _, err = dc.Resource(certGVR).Namespace("default").Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + Expect(err).ShouldNot(BeNil()) + }) + }) +} + +// 测试默认值设置 func CreateIngressMsbDeploymentDefaultValue(ctx *framework.TestContext, f *framework.Framework) { var ( // 1. 准备测试数据 diff --git a/test/e2e/create/testdata/create-ingress-with-tls.yaml b/test/e2e/create/testdata/create-ingress-with-tls.yaml new file mode 100644 index 0000000..33b063d --- /dev/null +++ b/test/e2e/create/testdata/create-ingress-with-tls.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.mashibing.com/v1 +kind: MsbDeployment +metadata: + name: create-ingress-with-tls +spec: + image: nginx + port: 80 + replicas: 2 + expose: + mode: ingress + ingressDomain: www.mashingbing-test-t.com + tls: true \ No newline at end of file diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 3cbe72d..7586cdc 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -6,6 +6,7 @@ package e2e import "mashibing.com/pkg/mashibing-deployment/test/e2e/create" var _ = fmw.Describe("Create msbdeployment mod ingress", create.CreateIngressMsbDeployment) +var _ = fmw.Describe("Create msbdeployment mod ingress with tls", create.CreateIngressMsbDeploymentWithTls) var _ = fmw.Describe("Create msbdeployment mod nodeport", create.CreateNodeportMsbDeployment) var _ = fmw.Describe("Create msbdeployment mod ingress default value", create.CreateIngressMsbDeploymentDefaultValue) var _ = fmw.Describe("Create msbdeployment mod ingress must failed", create.CreateIngressMsbDeploymentMustFailed) diff --git a/test/e2e/e2e.xml b/test/e2e/e2e.xml index 8bf95a7..4e9fb31 100644 --- a/test/e2e/e2e.xml +++ b/test/e2e/e2e.xml @@ -1,55 +1,67 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file