diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..cc9cc23448ee55b7fcac6cdde06ec7d266fdd5cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +__pycache__ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.pyc +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +.idea/ +.git/ +.gitee/ +logs/ +lastupdate.tmp diff --git a/cve-agency-manager/.gitignore b/cve-agency-manager/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a45f85f777eb9d95486eaf8a52318212bafefa8c --- /dev/null +++ b/cve-agency-manager/.gitignore @@ -0,0 +1,20 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.pyc +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +.idea/ +.git/ +.gitee/ +logs/ +lastupdate.tmp diff --git a/cve-agency-manager/Dockerfile b/cve-agency-manager/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..66d92c53c7d48ae25e12b4a6e9962185be457dc8 --- /dev/null +++ b/cve-agency-manager/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:latest as BUILDER +LABEL maintainer="zhangjianjun" + +# build binary +RUN mkdir -p /go/src/gitee.com/openeuler/cve-agency-manager +COPY . /go/src/gitee.com/openeuler/cve-agency-manager +RUN cd /go/src/gitee.com/openeuler/cve-agency-manager && CGO_ENABLED=1 go build -v -o ./cve-agency-manager main.go + +# copy binary config and utils +FROM openeuler/openeuler:21.03 +RUN yum update && yum install -y python3 && yum install -y python3-pip +RUN mkdir -p /opt/app/ && mkdir -p /opt/app/conf/ +COPY ./conf/product.conf /opt/app/conf/app.conf +# overwrite config yaml +COPY ./cve_tracking /opt/app/cve_tracking +RUN chmod 755 -R /opt/app/cve_tracking +RUN pip3 install --no-cache-dir -r /opt/app/cve_tracking/requirements.txt +COPY --from=BUILDER /go/src/gitee.com/openeuler/cve-agency-manager/cve-agency-manager /opt/app/cve-agency-manager + +WORKDIR /opt/app/ +ENTRYPOINT ["/opt/app/cve-agency-manager"] diff --git a/cve-agency-manager/common/errorcode.go b/cve-agency-manager/common/errorcode.go new file mode 100644 index 0000000000000000000000000000000000000000..d405c94fe1cd6086e2383cecca44536dce0d5cf5 --- /dev/null +++ b/cve-agency-manager/common/errorcode.go @@ -0,0 +1,54 @@ +package common + +const ( + RecodeOk = "200" + RecodeDbErr = "4001" + RecodeNodata = "4002" + RecodeDataExist = "4003" + RecodeDataErr = "4004" + RecodeSessionErr = "4101" + RecodeLoginErr = "4102" + RecodeParamErr = "4103" + RecodeUserErr = "4104" + RecodeRoleErr = "4105" + RecodePwdErr = "4106" + RecodeReqErr = "4201" + RecodeIpErr = "4202" + RecodeThirdErr = "4301" + RecodeIoErr = "4302" + RecodeServerErr = "4500" + RecodeUnknowErr = "4501" + RecodeDataMissErr = "4502" + RecodeCmdFailed = "4503" +) + +var recodeText = map[string]string{ + RecodeOk: "成功", + RecodeDbErr: "数据库查询错误", + RecodeNodata: "无数据", + RecodeDataExist: "数据已存在", + RecodeDataErr: "数据错误", + RecodeSessionErr: "用户未登录", + RecodeLoginErr: "用户登录失败", + RecodeParamErr: "请求参数错误", + RecodeUserErr: "用户不存在或未激活", + RecodeRoleErr: "登录身份错误", + RecodePwdErr: "密码错误", + RecodeReqErr: "非法请求或请求次数受限", + RecodeIpErr: "IP受限", + RecodeThirdErr: "第三方系统错误", + RecodeIoErr: "文件读写错误", + RecodeServerErr: "内部错误", + RecodeUnknowErr: "未知错误", + RecodeDataMissErr: "数据存在缺失(请检查cve编号、组件、版本号等)", + RecodeCmdFailed: "cmd调用失败", +} + +//RecodeText get resp string by code +func RecodeText(code string) string { + str, ok := recodeText[code] + if ok { + return str + } + return RecodeText(RecodeUnknowErr) +} diff --git a/cve-agency-manager/common/logs.go b/cve-agency-manager/common/logs.go new file mode 100644 index 0000000000000000000000000000000000000000..5dcc0a4b3f86273f3312b5171416f2b4b678ec6e --- /dev/null +++ b/cve-agency-manager/common/logs.go @@ -0,0 +1,47 @@ +package common + +import ( + "encoding/json" + "fmt" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" +) + +func InitLogger() (err error) { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + fmt.Println("config init error:", err) + return + } + maxlines, lerr := BConfig.Int64("log::maxlines") + if lerr != nil { + maxlines = 20000 + } + + logConf := make(map[string]interface{}) + logConf["filename"] = BConfig.String("log::log_path") + level, _ := BConfig.Int("log::log_level") + logConf["level"] = level + logConf["maxlines"] = maxlines + + confStr, err := json.Marshal(logConf) + if err != nil { + fmt.Println("marshal failed,err:", err) + return + } + err = logs.SetLogger(logs.AdapterFile, string(confStr)) + if err != nil { + fmt.Println("marshal failed,err:", err) + return + } + logs.SetLogFuncCall(true) + return +} + +func LogInit() { + err := InitLogger() + if err != nil { + fmt.Println(err) + } + fmt.Println("log init success !") +} diff --git a/cve-agency-manager/conf/app.conf b/cve-agency-manager/conf/app.conf new file mode 100644 index 0000000000000000000000000000000000000000..2476e5b4c3735b920cd83bda60b9b31d8583102f --- /dev/null +++ b/cve-agency-manager/conf/app.conf @@ -0,0 +1,17 @@ +appname = cve-agency-manager +httpport = 8090 +runmode = dev +autorender = false +copyrequestbody = true +EnableDocs = true +sqlconn = + +[log] +log_level = 7 +log_dir = ./logs +log_path = logs/cve.log +maxlines=25000 +maxsize=204800 + +[python_conf] +python_file = cve_tracking/main.py \ No newline at end of file diff --git a/cve-agency-manager/conf/product.conf b/cve-agency-manager/conf/product.conf new file mode 100644 index 0000000000000000000000000000000000000000..899ef06a375d085615620c790f2df7808d39b1d0 --- /dev/null +++ b/cve-agency-manager/conf/product.conf @@ -0,0 +1,17 @@ +appname = cve-manager-agent +httpport = 80 +runmode = dev +autorender = false +copyrequestbody = true +EnableDocs = true +sqlconn = + +[log] +log_level = 5 +log_dir = ./logs +log_path = logs/cve.log +maxlines=25000 +maxsize=204800 + +[python_conf] +python_file = /opt/app/cve_tracking/main.py \ No newline at end of file diff --git a/cve-agency-manager/controllers/default.go b/cve-agency-manager/controllers/default.go new file mode 100644 index 0000000000000000000000000000000000000000..9a9e6f2454076ac3c49b8395afd0dea3d1be0d75 --- /dev/null +++ b/cve-agency-manager/controllers/default.go @@ -0,0 +1,15 @@ +package controllers + +import ( + "github.com/astaxie/beego" +) + +type MainController struct { + beego.Controller +} + +func (c *MainController) Get() { + c.Data["Website"] = "beego.me" + c.Data["Email"] = "astaxie@gmail.com" + c.TplName = "index.tpl" +} diff --git a/cve-agency-manager/controllers/live.go b/cve-agency-manager/controllers/live.go new file mode 100644 index 0000000000000000000000000000000000000000..1c77344bcbf5f4ef1ba1aac13c2b46a5cb90281d --- /dev/null +++ b/cve-agency-manager/controllers/live.go @@ -0,0 +1,51 @@ +package controllers + +import ( + "github.com/astaxie/beego" +) + +type TriggerHeartbeatreadController struct { + beego.Controller +} + +func (c *TriggerHeartbeatreadController) RetCveData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title Heartbeatreadiness +// @Description Heartbeatreadiness +// @Param body body models.Heartbeatreadiness true +// @Success 200 {int} models.Id +// @Failure 403 body is empty +// @router / [get] +func (u *TriggerHeartbeatreadController) Get() { + resp := make(map[string]interface{}) + resp["code"] = "200" + resp["errmsg"] = "success" + resp["body"] = "success" + defer u.RetCveData(resp) +} + +type TriggerHeartbeatLiveController struct { + beego.Controller +} + +func (c *TriggerHeartbeatLiveController) RetCveData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title HeartbeatLive check +// @Description get HeartbeatLive +// @Param body body models.Heartbeatreadiness true +// @Success 200 {int} +// @Failure 403 :status is err +// @router / [get] +func (u *TriggerHeartbeatLiveController) Get() { + resp := make(map[string]interface{}) + resp["code"] = "200" + resp["errmsg"] = "success" + resp["body"] = "success" + defer u.RetCveData(resp) +} diff --git a/cve-agency-manager/controllers/object.go b/cve-agency-manager/controllers/object.go new file mode 100644 index 0000000000000000000000000000000000000000..ee28d544312a0ad121b7f4435e14f55eada3a6e1 --- /dev/null +++ b/cve-agency-manager/controllers/object.go @@ -0,0 +1,91 @@ +package controllers + +import ( + "cve-agency-manager/models" + "encoding/json" + "github.com/astaxie/beego" +) + +// Operations about object +type ObjectController struct { + beego.Controller +} + +// @Title Create +// @Description create object +// @Param body body models.Object true "The object content" +// @Success 200 {string} models.Object.Id +// @Failure 403 body is empty +// @router / [post] +func (o *ObjectController) Post() { + var ob models.Object + json.Unmarshal(o.Ctx.Input.RequestBody, &ob) + objectid := models.AddOne(ob) + o.Data["json"] = map[string]string{"ObjectId": objectid} + o.ServeJSON() +} + +// @Title Get +// @Description find object by objectid +// @Param objectId path string true "the objectid you want to get" +// @Success 200 {object} models.Object +// @Failure 403 :objectId is empty +// @router /:objectId [get] +func (o *ObjectController) Get() { + objectId := o.Ctx.Input.Param(":objectId") + if objectId != "" { + ob, err := models.GetOne(objectId) + if err != nil { + o.Data["json"] = err.Error() + } else { + o.Data["json"] = ob + } + } + o.ServeJSON() +} + +// @Title GetAll +// @Description get all objects +// @Success 200 {object} models.Object +// @Failure 403 :objectId is empty +// @router / [get] +func (o *ObjectController) GetAll() { + obs := models.GetAll() + o.Data["json"] = obs + o.ServeJSON() +} + +// @Title Update +// @Description update the object +// @Param objectId path string true "The objectid you want to update" +// @Param body body models.Object true "The body" +// @Success 200 {object} models.Object +// @Failure 403 :objectId is empty +// @router /:objectId [put] +func (o *ObjectController) Put() { + objectId := o.Ctx.Input.Param(":objectId") + var ob models.Object + json.Unmarshal(o.Ctx.Input.RequestBody, &ob) + + err := models.Update(objectId, ob.Score) + if err != nil { + o.Data["json"] = err.Error() + } else { + o.Data["json"] = "update success!" + } + o.ServeJSON() +} + +// @Title Delete +// @Description delete the object +// @Param objectId path string true "The objectId you want to delete" +// @Success 200 {string} delete success! +// @Failure 403 objectId is empty +// @router /:objectId [delete] +func (o *ObjectController) Delete() { + objectId := o.Ctx.Input.Param(":objectId") + models.Delete(objectId) + o.Data["json"] = "delete success!" + o.ServeJSON() +} + diff --git a/cve-agency-manager/controllers/track.go b/cve-agency-manager/controllers/track.go new file mode 100644 index 0000000000000000000000000000000000000000..c85e1630d4ea0a9c6c071bde9d02f31b7c60eb22 --- /dev/null +++ b/cve-agency-manager/controllers/track.go @@ -0,0 +1,68 @@ +package controllers + +import ( + "cve-agency-manager/common" + "encoding/json" + "os/exec" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" +) + +type TriggerTrackController struct { + beego.Controller +} + +func (c *TriggerTrackController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type CveReqPramData struct { + CveNum string `json:"cveNum"` + PackName string `json:"packName"` + PackVersion string `json:"packVersion"` + IssueNumber string `json:"issueNumber"` + AffectBranch string `json:"affectBranch"` + CveUrl string `json:"cveUrl"` + PatchUrl string `json:"patchUrl"` +} + +// @Title TriggerTrack +// @Description Trigger Track +// @Param body body models.TriggerTrack true +// @Success 200 {int} models.Id +// @Failure 403 body is empty +// @router / [post] +func (c *TriggerTrackController) Post() { + var cvePram CveReqPramData + req := c.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, ",Header: ", req.Header) + resp := make(map[string]interface{}) + resp["errno"] = common.RecodeParamErr + resp["errmsg"] = common.RecodeText(common.RecodeParamErr) + defer c.RetData(resp) + json.Unmarshal(c.Ctx.Input.RequestBody, &cvePram) + logs.Info("trigger track request parameters: ", string(c.Ctx.Input.RequestBody)) + // Call python script + pythonFile := beego.AppConfig.String("python_conf::python_file") + args := []string{} + if len(cvePram.CveUrl) > 1 || len(cvePram.PatchUrl) > 1 { + args = []string{pythonFile, "feedback", "-c", cvePram.CveNum, "-p", cvePram.CveUrl, "-u", cvePram.PatchUrl} + } else { + args = []string{pythonFile, "comment", "-c", cvePram.CveNum, "-r", cvePram.PackName, "-i", cvePram.IssueNumber} + } + //args := []string{pythonFile} + out, err := exec.Command("python3", args...).Output() + if err != nil { + logs.Error("cmd python3 ==> python ", args, ",err: ", err) + resp["errno"] = common.RecodeCmdFailed + resp["errmsg"] = common.RecodeText(common.RecodeCmdFailed) + return + } + logs.Info("out: ", string(out)) + resp["errno"] = common.RecodeOk + resp["errmsg"] = common.RecodeText(common.RecodeOk) + return +} diff --git a/cve-agency-manager/controllers/user.go b/cve-agency-manager/controllers/user.go new file mode 100644 index 0000000000000000000000000000000000000000..b473a659719fba4dc8921c0300e4fc0446550b53 --- /dev/null +++ b/cve-agency-manager/controllers/user.go @@ -0,0 +1,118 @@ +package controllers + +import ( + "cve-agency-manager/models" + "encoding/json" + "github.com/astaxie/beego" +) + +// Operations about Users +type UserController struct { + beego.Controller +} + +// @Title CreateUser +// @Description create users +// @Param body body models.User true "body for user content" +// @Success 200 {int} models.User.Id +// @Failure 403 body is empty +// @router / [post] +func (u *UserController) Post() { + var user models.User + json.Unmarshal(u.Ctx.Input.RequestBody, &user) + uid := models.AddUser(user) + u.Data["json"] = map[string]string{"uid": uid} + u.ServeJSON() +} + +// @Title GetAll +// @Description get all Users +// @Success 200 {object} models.User +// @router / [get] +func (u *UserController) GetAll() { + users := models.GetAllUsers() + u.Data["json"] = users + u.ServeJSON() +} + +// @Title Get +// @Description get user by uid +// @Param uid path string true "The key for staticblock" +// @Success 200 {object} models.User +// @Failure 403 :uid is empty +// @router /:uid [get] +func (u *UserController) Get() { + uid := u.GetString(":uid") + if uid != "" { + user, err := models.GetUser(uid) + if err != nil { + u.Data["json"] = err.Error() + } else { + u.Data["json"] = user + } + } + u.ServeJSON() +} + +// @Title Update +// @Description update the user +// @Param uid path string true "The uid you want to update" +// @Param body body models.User true "body for user content" +// @Success 200 {object} models.User +// @Failure 403 :uid is not int +// @router /:uid [put] +func (u *UserController) Put() { + uid := u.GetString(":uid") + if uid != "" { + var user models.User + json.Unmarshal(u.Ctx.Input.RequestBody, &user) + uu, err := models.UpdateUser(uid, &user) + if err != nil { + u.Data["json"] = err.Error() + } else { + u.Data["json"] = uu + } + } + u.ServeJSON() +} + +// @Title Delete +// @Description delete the user +// @Param uid path string true "The uid you want to delete" +// @Success 200 {string} delete success! +// @Failure 403 uid is empty +// @router /:uid [delete] +func (u *UserController) Delete() { + uid := u.GetString(":uid") + models.DeleteUser(uid) + u.Data["json"] = "delete success!" + u.ServeJSON() +} + +// @Title Login +// @Description Logs user into the system +// @Param username query string true "The username for login" +// @Param password query string true "The password for login" +// @Success 200 {string} login success +// @Failure 403 user not exist +// @router /login [get] +func (u *UserController) Login() { + username := u.GetString("username") + password := u.GetString("password") + if models.Login(username, password) { + u.Data["json"] = "login success" + } else { + u.Data["json"] = "user not exist" + } + u.ServeJSON() +} + +// @Title logout +// @Description Logs out current logged in user session +// @Success 200 {string} logout success +// @router /logout [get] +func (u *UserController) Logout() { + u.Data["json"] = "logout success" + u.ServeJSON() +} + diff --git a/cve-agency-manager/cve_tracking/.gitignore b/cve-agency-manager/cve_tracking/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..11614af2870733183efe883810764d8708bddf8f --- /dev/null +++ b/cve-agency-manager/cve_tracking/.gitignore @@ -0,0 +1,115 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cve-agency-manager/cve_tracking/README.en.md b/cve-agency-manager/cve_tracking/README.en.md new file mode 100644 index 0000000000000000000000000000000000000000..cf82884087434fbcf50128696c2796e50adb1eee --- /dev/null +++ b/cve-agency-manager/cve_tracking/README.en.md @@ -0,0 +1,36 @@ +# cve_tracking + +#### Description +cve补丁自动获取工具 + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/cve-agency-manager/cve_tracking/README.md b/cve-agency-manager/cve_tracking/README.md new file mode 100644 index 0000000000000000000000000000000000000000..45d962b3a2b9634e06951249eae032abf782c621 --- /dev/null +++ b/cve-agency-manager/cve_tracking/README.md @@ -0,0 +1,123 @@ +# cve_tracking + +#### 介绍 + +cve 补丁自动获取工具，该工具将会根据 cve 和 rpm 包自动在上游社区查找补丁并反馈结果，同时也可以下载查找到的补丁以及验证补丁的可用性。 + +#### 软件架构 + +python 可执行代码 + +#### 安装教程 + +1. 下载代码 + + ```shell + git clone https://gitee.com/openeuler/cve-manager.git + ``` + +2. 进入工具执行目录 + + ```shell + cd xxx(上述代码下载目录)/cve-manager/cve-agency-manager/cve_tracking + ``` + +3. 在 cve-tracking.yaml 的 authentication 中设置 gitee（gitee 的私人令牌）和 gitlab（gitlab 的私人令牌，默认设置了一个临时令牌，可临时使用），github（github 的私人令牌）可不设置。 + +4. 安装依赖包 + + ``` + pip3 install -r requirements.txt + ``` + +5. 根据使用说明执行工具 + +#### 使用说明 + +1. 补丁查找及评论 issue + + ```shell + python3 main.py comment -c cve_num -r rpm_name -i issue_num + ``` + + > 参数说明： + > + > -c cve 的编号 + > + > -r rpm 包名称 + > + > -i 需要评论的 issue 编号 + > + > 注意：默认仓库为 src-openeuler，如果要更改，请修改 main.py 同目录下 constant.py 中的 DEFAULT_OWNER 的值。 + +2. 补丁查找及下载(验证) + + ``` + python3 main.py download -c cve_num -r rpm_name [-f patch_save_path] [-s source_path] [-p] [-b branch] + ``` + + > 参数说明： + > + > -c cve 的编号 + > + > -r rpm 包名称 + > + > -f 补丁文件的下载目录，不设置默认为/opt/cve_tracking/patches + > + > -s 源码包下载路径，不设置默认为/opt/cve_tracking/source_code + > + > -b 源码包所在的 gitee 的 src-openeuler 仓库的分支，默认为 master + > + > -p 是否进行补丁应用，默认为不应用，若需要应用，添加该参数。 + +3. 补丁验证 + + ``` + python3 main.py packing -r rpm_name -f patch_save_path -s source_path -b branch [-nd] + ``` + + > 参数说明： + > + > -r rpm 包名称 + > + > -f 补丁文件路径（指定到补丁所在的文件夹即可） + > + > -s 源码包路径，如果无需下载指定为本地源码包的路径；如果需要下载指定为需要下载源码包的路径即可 + > + > -b 源码包所在 gitee 中 src-openeuler 仓库的分支，不设置默认为 master + > + > -nd 是否需要下载源代码，默认为需要下载，若无需下载添加该参数 + +4. 意见反馈 + + 若工具没有查找到补丁且人工查找到补丁信息，可通过该功能进行反馈，将会在配置的平台中提交 issue。将发现新 patch 的平台信息上报至指定的仓库，便于 CVE 修复人员完善工具的查找。指定的仓库可以在 conf.ini 配置文件的【FEEDBACK】中进行配置 + + ```shell + python3 main.py feedback -c cve_num -p cve_platform -u patch_url -i issue_warehouse + ``` + + > 参数说明： + > + > -c CVE 的编号 + > + > -p 查找 CVE 所在的平台，可按照平台名称简写 + > + > -u 修复 CVE 的补丁链接 + > + > -i 管理 issue 的仓库，只能是（gitee、gitlab、github）中任选其一 + +#### 参与贡献 + +1. Fork 本仓库 +2. 新建 Feat_xxx 分支 +3. 提交代码 +4. 新建 Pull Request + +#### 特技 + +1. 使用 Readme_XXX.md 来支持不同的语言，例如 Readme_en.md, Readme_zh.md +2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) +3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 +4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 +5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) +6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/cve-agency-manager/cve_tracking/cli/__init__.py b/cve-agency-manager/cve_tracking/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..41ef326fdecdf0c1e0a9812ed47acd15b8fd5d74 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from .base import CveTrackingCommand +from .comment import CommentCommand +from .feedback import FeedbackCommand +from .save import SaveCommand +from .verify import VerifyCommand + +__all__ = ("CveTrackingCommand", + "CommentCommand", + "SaveCommand", + "VerifyCommand", + "FeedbackCommand") diff --git a/cve-agency-manager/cve_tracking/cli/base.py b/cve-agency-manager/cve_tracking/cli/base.py new file mode 100644 index 0000000000000000000000000000000000000000..a92dcc3c4aa6a6dac8e7e2f303d8ef3f467367f7 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/base.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import argparse + + +class CveTrackingCommand: + """ + Cve tool command line related classes + """ + arg_parse = argparse.ArgumentParser(description="Cve_tracking Tool") + sub_parse = arg_parse.add_subparsers(description="sub-command ") + + def __init__(self): + self.parser = None + + def _add_common_param(self): + """ + Add public parameters and set execution method to subcommands + :return: None + """ + self.parser.add_argument('-c', '--cve_num', metavar='cve', type=str, required=True, action='store', + default=None, help='cve number') + self.parser.add_argument('-r', '--rpm_name', metavar='rpm', type=str, required=True, action='store', + default=None, help='rpm name') + + @staticmethod + def register_command(sub_instance): + """ + Initialize each sub-command class and set the execution method + :param sub_instance: sub class instance + :return: None + """ + sub_instance.parser.set_defaults(func=sub_instance.run_command) + + @classmethod + async def run(cls): + """ + Command line execution entry + :return: None + """ + args = cls.arg_parse.parse_args() + await args.func(args) diff --git a/cve-agency-manager/cve_tracking/cli/comment.py b/cve-agency-manager/cve_tracking/cli/comment.py new file mode 100644 index 0000000000000000000000000000000000000000..b45186377c580cd21e2b634f98cf5afd0776ca5e --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/comment.py @@ -0,0 +1,63 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from cli.base import CveTrackingCommand +from conf import CONFIG +from core.comment.issue_comment import issue_comment +from core.crawler.patch import Patch +from exception import InputError +from logger import logger +from util.gitee_api import Gitee + + +class CommentCommand(CveTrackingCommand): + """ + Comment function command line + """ + + def __init__(self): + super(CommentCommand, self).__init__() + self.parser = CveTrackingCommand.sub_parse.add_parser( + "comment", help="Add a comment to the issue on gitee" + ) + self._add_common_param() + self.parser.add_argument( + "-i", + "--issue", + metavar="issue", + type=str, + required=True, + action="store", + default=None, + help="CVE related issue number", + ) + + @staticmethod + async def run_command(args): + """ + Comment function execution entrance + :param args: command line params + :return: None + """ + logger.info("Start to perform search and comment functions") + if not all([args.cve_num, args.rpm_name, args.issue]): + raise InputError(msg="Comment command parameters: cve_num/rpm_name/issue") + + # find patch detail + patch_obj = Patch(cve_num=args.cve_num, rpm_name=args.rpm_name) + patch_details = await patch_obj.find_patches_detail() + + # comment cve issue + gitee = Gitee() + gitee.set_attr(owner=CONFIG.DEFAULT_OWNER, repo=args.rpm_name) + await issue_comment(patch_details=patch_details, number=args.issue, gitee=gitee) + logger.info("End to perform search and comment functions") diff --git a/cve-agency-manager/cve_tracking/cli/feedback.py b/cve-agency-manager/cve_tracking/cli/feedback.py new file mode 100644 index 0000000000000000000000000000000000000000..1bd42ff5dd67946ef5c1269a6aa712fe9c458652 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/feedback.py @@ -0,0 +1,112 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from cli import CveTrackingCommand +from conf import CONFIG +from constant import Constant +from core.feedback.issue_mode import IssueMode +from exception import InputError +from logger import logger + + +class FeedbackCommand(CveTrackingCommand): + """ + Cve tool feedback function command line + """ + + def __init__(self): + super(FeedbackCommand, self).__init__() + self.parser = CveTrackingCommand.sub_parse.add_parser( + name="feedback", + help="Submit an issue to the designated warehouse, and the feedback tool will find the result", + ) + self._add_params() + + def _add_params(self): + """ + Add input parameters required for feedback function + :return: None + """ + self.parser.add_argument( + "-c", + "--cve_num", + metavar="cve", + type=str, + required=True, + action="store", + default=None, + help="cve number", + ) + self.parser.add_argument( + "-p", + "--cve_platform", + type=str, + metavar="cve_platform", + required=True, + action="store", + default=None, + help="Correct cve reference URL", + ) + self.parser.add_argument( + "-u", + "--patch_url", + type=str, + metavar="patch_url", + required=True, + action="store", + default=None, + help="Correct patch url", + ) + self.parser.add_argument( + "-i", + "--issue_platform", + type=str, + metavar="issue_platform", + required=False, + action="store", + choices=Constant.CODE_PLATFORM, + default=CONFIG.FEEDBACK_PLATFORM, + help="Issue submission platform, default is gitee", + ) + + @staticmethod + async def run_command(args): + """ + Feedback function execution entrance + :param args: input params + :return: None + """ + logger.info( + f"Start add issue about cve:{args.cve_num} on platform:{args.issue_platform}" + ) + if not all( + [args.cve_num, args.cve_platform, args.patch_url, args.issue_platform] + ): + raise InputError(msg="Feedback command params, please check and try again.") + + issue_mode = IssueMode( + cve_num=args.cve_num, + cve_platform=args.cve_platform, + patch_url=args.patch_url, + issue_platform=args.issue_platform, + ) + result = await issue_mode.create_feedback_issue() + + if result: + logger.info( + f"Success to add issue about cve:{args.cve_num} on platform:{args.issue_platform}" + ) + else: + logger.error( + f"Failed to add issue about cve:{args.cve_num} on platform:{args.issue_platform}, " + f"content is cve_platform:{args.cve_platform}, patch_url:{args.patch_url}" + ) diff --git a/cve-agency-manager/cve_tracking/cli/save.py b/cve-agency-manager/cve_tracking/cli/save.py new file mode 100644 index 0000000000000000000000000000000000000000..2d9d5b8bc50c307705be1f1dfd661fc7d1ae5ad2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/save.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from cli.base import CveTrackingCommand +from conf import CONFIG +from core.crawler.patch import Patch +from core.download import save +from core.verification.apply import PathApply +from exception import InputError +from logger import logger + + +class SaveCommand(CveTrackingCommand): + """ + Execute the command line to find the patch and save the patch file + """ + + def __init__(self): + super(SaveCommand, self).__init__() + self.parser = CveTrackingCommand.sub_parse.add_parser('download', + help="Perform a patch search and download the patch file") + self._add_common_param() + self._add_params() + + def _add_params(self): + """ + Add the unique parameters of the command + :return: None + """ + self.parser.add_argument('-f', '--patch_path', metavar='patch_patch', type=str, action='store', required=False, + default=CONFIG.PATCH_SAVE_PATH, + help='Path to save the patch file, the default is"/opt/cve_tracking/patches"') + self.parser.add_argument('-s', '--source_path', metavar='source_patch', type=str, action='store', + required=False, default=CONFIG.SOURCE_CODE_PATH, + help='Source download path, the default is: "/opt/cve_tracking/source_code"') + self.parser.add_argument('-p', '--packing', action='store_true', required=False, default=False, + help='Whether to package the verification patch') + self.parser.add_argument('-b', '--branch', metavar='branch', type=str, action='store', required=False, + default='master', help='Branch of rpm, default is "master"') + + @staticmethod + async def run_command(args): + """ + Search patch, save file function execution entry + :param args: command line params + :return: None + """ + logger.info(f'Start to execute the download command, cve: {args.cve_num}, ' + f'rpm: {args.rpm_name}, patch save path: {args.patch_path}') + if not all([args.rpm_name, args.cve_num, args.patch_path]): + raise InputError('Download command parameter error, please confirm "rpm_name"/cve_num/patch_path"') + if args.packing and not all([args.source_path, args.branch]): + raise InputError('Download command parameter error, please confirm "source_path/branch"') + + # find patch info + patch = Patch(cve_num=args.cve_num, rpm_name=args.rpm_name) + patch_detail_info = await patch.find_patches_detail() + # download patch file + save.save_patch(patch_details=patch_detail_info, cve=args.cve_num, path=args.patch_path) + logger.info(f'End to execute the save file, patch save path:{args.patch_path}') + + # apply patch + if args.packing: + patch_apply = PathApply(rpm_name=args.rpm_name, patch_path=args.patch_path, + source_path=args.source_path, branch_rpm=args.branch) + patch_apply.packing_source() diff --git a/cve-agency-manager/cve_tracking/cli/verify.py b/cve-agency-manager/cve_tracking/cli/verify.py new file mode 100644 index 0000000000000000000000000000000000000000..ace288d8adce701b7790a1ca0321c1e8e330995a --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/verify.py @@ -0,0 +1,56 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from cli.base import CveTrackingCommand +from conf import CONFIG +from core.verification.apply import PathApply + + +class VerifyCommand(CveTrackingCommand): + """ + Patch verification function command line + """ + + def __init__(self): + super(VerifyCommand, self).__init__() + self.parser = CveTrackingCommand.sub_parse.add_parser('packing', help='Test patch file application results') + self._add_params() + + def _add_params(self): + """ + Add the unique parameters of the command + :return: None + """ + self.parser.add_argument('-r', '--rpm_name', metavar='rpm_name', type=str, required=True, default=None, + help='Source rpm name') + self.parser.add_argument('-f', '--patch_path', metavar='patch_path', type=str, action='store', required=True, + default=None, help='Path of patch file') + self.parser.add_argument('-s', '--source_path', metavar='source_path', type=str, action='store', required=False, + default=CONFIG.SOURCE_CODE_PATH, + help='Source download path, the default is: "/opt/cve_tracking/source_code"') + self.parser.add_argument('-b', '--branch', metavar='branch', type=str, action='store', required=False, + default='master', help='Branch of rpm, default is "master"') + self.parser.add_argument('-nd', '--no_download', action='store_true', required=False, default=False, + help='Do you need to download the source code? if you choose False, ' + '"source path" must be specified to the local source code repository') + + @staticmethod + async def run_command(args): + """ + Patch test application execution entry + :param args: command line params + :return: None + """ + branch = "no" if args.no_download else args.branch + patch_apply = PathApply(rpm_name=args.rpm_name, branch_rpm=branch, patch_path=args.patch_path, + source_path=args.source_path) + patch_apply.packing_source() diff --git a/cve-agency-manager/cve_tracking/conf/__init__.py b/cve-agency-manager/cve_tracking/conf/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..683c506a89390bb5cd51fb2ad98d1e29fde7a4f3 --- /dev/null +++ b/cve-agency-manager/cve_tracking/conf/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from .setting import Config, YamlConfiguration + + +CONFIG = Config() +settings = YamlConfiguration() + +__all__ = ("CONFIG", "settings") diff --git a/cve-agency-manager/cve_tracking/conf/setting.py b/cve-agency-manager/cve_tracking/conf/setting.py new file mode 100644 index 0000000000000000000000000000000000000000..d9a5152c628cbccd4d8edc68081d1be7d070c5dc --- /dev/null +++ b/cve-agency-manager/cve_tracking/conf/setting.py @@ -0,0 +1,178 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import configparser +import os +import yaml +from exception import ConfigNotFoundError + + +class DefaultConfig: + """ + Configuration items that all users may need to modify + """ + + # ********** + # Log related + # ********** + # level of LOG + LOGGER_LEVEL = "INFO" + # Log file path + LOGGER_PATH = "/var/log/cve_tracking" + # Log file name + LOGGER_FILE_NAME = "cve_tracking.log" + # Log file size, the default is 5 M + LOGGER_BUFFER = 5242880 + # Number of log files saved + LOGGER_FILE_COUNT = 20 + + # ********** + # Path related + # ********** + # Patch file download and save path. + PATCH_SAVE_PATH = "/opt/cve_tracking/patches" + # Source code package and patch file storage path, + # used to download gitee source code package and package verification. + SOURCE_CODE_PATH = "/opt/cve_tracking/source_code" + + # ********** + # Feedback issue related configuration + # ********** + + FEEDBACK_PLATFORM = "github" + # Feedback to the owner of the issue + FEEDBACK_ISSUE_OWNER = "liheavy" + # Feedback to the repo of the issue + FEEDBACK_ISSUE_REPO = "cve_tracking" + + # The owner of the comment function submits the comment + DEFAULT_OWNER = "src-openeuler" + + +class Config: + """ + Configuration item load and get value + """ + + config_file = os.path.join( + os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "config.ini" + ) + + def __init__(self): + self._setattr_default() + self._setattr_customize() + + def _setattr_default(self): + """ + Set default configuration items and their values + :return: self + """ + for attr in dir(DefaultConfig): + if not str(attr).startswith("_"): + setattr(self, attr, getattr(DefaultConfig, attr)) + + def _setattr_customize(self): + """ + Set customize configuration items and their values + :return: self + """ + if not os.path.exists(self.config_file): + raise ConfigNotFoundError(self.config_file) + + conf_parser = configparser.ConfigParser() + conf_parser.read(self.config_file) + for section in conf_parser.sections(): + for key_value in conf_parser.items(section): + if key_value[1]: + setattr(self, key_value[0].upper(), key_value[1]) + + def __getattr__(self, item): + """ + When the configuration cannot be found, reload the configuration file + :param item: config key + :return: value + """ + self._setattr_default() + self._setattr_customize() + try: + return self.__dict__[item] + except KeyError: + return None + + +class YamlConfiguration: + """ + yaml file parsing class + """ + yaml = os.path.join( + os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "cve-tracking.yaml" + ) + + def __init__(self) -> None: + if not os.path.exists(self.yaml): + raise FileNotFoundError("file not found: %s" % self.yaml) + self.__dict__.update(self._parse_yaml()) + + def _parse_yaml(self): + with open(self.yaml, "r", encoding="utf-8") as file: + try: + configs = yaml.load(file.read(), Loader=yaml.FullLoader) + except yaml.YAMLError as error: + raise ValueError( + "The format of the yaml configuration " + "file is wrong please check and try again:{0}".format(error) + ) + else: + return configs + + def __getattr__(self, name): + if name not in self.__dict__: + return None + + return self.__dict__[name] + + def get_platform(self, name=None): + """ + Get the configured cve platform + :param name: specify platform name + :return: platform name + """ + if name is None: + return self.platform + platform = filter(lambda x: x["name"] == name, self.platform) + if platform: + return list(platform)[-1] + + def get_regex(self, label=None): + """ + Regular expression to get patch + :param label: issue or pr or commit + :return: regex + """ + if label is None: + regulars = [reg for reg in self.regex] + else: + regulars = filter(lambda x: x["label"] == label, self.regex) + return [regx for reg in regulars for regx in reg["regular"]] + + def token(self, name): + token = filter(lambda x: x["name"] == name, self.authentication) + if token: + return list(token)[-1]["token"] + + @property + def configuration(self): + """ + All configuration items + :return: dict + """ + return self.__dict__ diff --git a/cve-agency-manager/cve_tracking/config.ini b/cve-agency-manager/cve_tracking/config.ini new file mode 100644 index 0000000000000000000000000000000000000000..ccd7f5809e7a834c6673b7d29e00ee934f3ae169 --- /dev/null +++ b/cve-agency-manager/cve_tracking/config.ini @@ -0,0 +1,46 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +;Log related configuration +[LOG] +;level of LOG +LOGGER_LEVEL = INFO +; Log file path +LOGGER_PATH = /var/log/cve_tracking +; Log file name +LOGGER_FILE_NAME = log.info +; Log file size, the default is 5 M +LOGGER_BUFFER = 5242880 +; Number of log files saved +LOGGER_FILE_COUNT = 20 + +;Path related configuration +[PATH] +;Patch file download and save path. +PATCH_SAVE_PATH = /opt/cve_tracking/patches +;Source code package and patch file storage path, used to download gitee source code package and package verification. +SOURCE_CODE_PATH = /opt/cve_tracking/source_code + +;Comment function related configuration +[COMMENT] +;The owner of the comment function submits the comment +DEFAULT_OWNER = src-openeuler + +;Feedback issue related configuration +[FEEDBACK] +;Feedback issue platform,Currently supports "gitee" "github" "gitlab",default is "gitee" +;Please configure the token of the corresponding platform synchronously in cve-tracking.yaml +FEEDBACK_PLATFORM = gitee +;Feedback to the owner of the issue +FEEDBACK_ISSUE_OWNER = liheavy +;Feedback to the repo of the issue +FEEDBACK_ISSUE_REPO = cve_tracking \ No newline at end of file diff --git a/cve-agency-manager/cve_tracking/constant.py b/cve-agency-manager/cve_tracking/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..1bcd29c71c151722ac951a0fa62380db0a3c95dd --- /dev/null +++ b/cve-agency-manager/cve_tracking/constant.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ + + +class Constant: + """ + Constant class + """ + + # gitlab host format + GITLAB_HOST_REGEX = r"http[s]?://gitlab(?:[-\w.?])+" + """ + Code hosting platform related + """ + # github + GITHUB = "github" + # gitlab + GITLAB = "gitlab" + # Supported code hosting platforms + CODE_PLATFORM = ["gitee", "github", "gitlab"] + + """ + Other + """ + # The longest waiting time default + MAX_DELAY = 180 + # The longest waiting time for url calls + CALL_MAX_DELAY = 60 + # The longest waiting time for download + MAX_DOWNLOAD_DELAY = 120 + # Maximum number of retries for failed URL calls + MAX_RETRY = 3 diff --git a/cve-agency-manager/cve_tracking/core/__init__.py b/cve-agency-manager/cve_tracking/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e26da5218904e6406ea988ff828312ffb3f9fe2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/cve-agency-manager/cve_tracking/core/comment/__init__.py b/cve-agency-manager/cve_tracking/core/comment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e26da5218904e6406ea988ff828312ffb3f9fe2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/comment/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/cve-agency-manager/cve_tracking/core/comment/issue_comment.py b/cve-agency-manager/cve_tracking/core/comment/issue_comment.py new file mode 100644 index 0000000000000000000000000000000000000000..f5fdfbc467ee0b4b6b6e5302fa4496d305f35430 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/comment/issue_comment.py @@ -0,0 +1,125 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import re + +from util.gitee_api import Gitee + + +class Table: + @staticmethod + def _table(content): + return f"{content}
" + + @staticmethod + def _th(content): + return f"{content}" + + @staticmethod + def _tr(content): + return f"{content}" + + @staticmethod + def _td(content): + return f"{content}" + + @staticmethod + def _rowspan(num, platform): + return """{}""".format(str(num), platform) + + def span_content(self, pr_url, pr_status, commits): + """ + Splicing comment content + :param pr_url: pull request url + :param pr_status: pull request status + :param commits: commits + :return: content spliced + """ + hyperlink_commits = [f"{commit}" for commit in commits] + contents = self._td(pr_url) + self._td(pr_status) + self._td("
".join(hyperlink_commits)) + return contents + + def _table_name(self): + """ + Splicing header + :return: header + """ + table_names = ["参考网址", "关联pr", "状态", "补丁链接"] + table_names = " ".join(map(self._th, table_names)) + return self._tr(table_names) + + @staticmethod + def _row_span(detail): + number = 0 + for issues in detail: + number += len(issues["issue"].get("prs", list())) + return number + + def table(self, lst): + """ + Splicing table + :param lst: cve info + :return: table + """ + contents = "" + for platforms in lst: + platform_index = 0 + platform = platforms.get("platform") + if platform and re.search("bugzilla.redhat.com", platform): + platform = f"https://bugzilla.redhat.com/show_bug.cgi?id={platform.split('/')[-2]}" + details = platforms.get("details") + rowspan_content = self._rowspan(self._row_span(details), platform) + + if not details: + none_tr = self._tr(self._td(platform) + self._td("") * 3) + contents += none_tr + + for issue_info in details: + for pr in issue_info["issue"].get("prs", list()): + content = self.span_content( + pr["url"], + pr["status"], + pr.get("commits", []), + ) + if platform_index == 0: + rowspan_content = rowspan_content + content + span_content = self._tr(rowspan_content) + platform_index += 1 + else: + span_content = self._tr(content) + contents += span_content + + return self._table(self._table_name() + contents) + + +async def issue_comment(patch_details, number, gitee: Gitee): + """ + Call gitee's api to add issue comments + :param patch_details: patch detail + :param number: issue number + :param gitee: gitee instance + :return: None + """ + table = Table().table(patch_details) + instruction = "\n" * 2 + "> 说明：抱歉，当前工具暂未找到推荐补丁，请人工查找或者之后评论'/find-patch'尝试再次查找。\n" \ + "若人工查找到补丁，烦请在此issue下评论 '/report-patch 参考网址 补丁链接1,补丁链接2' 便于我们不断优化工具，不胜感激。\n" \ + "如 /report-patch https://security-tracker.debian.org/tracker/CVE-2021-3997 https://github.com/systemd/systemd/commit/5b1cf7a9be37e20133c0208005274ce4a5b5c6a1" + for cve_info in patch_details: + if cve_info.get("details"): + instruction = "\n" * 2 + "> 说明：补丁链接仅供初步排查参考，实际可用性请人工再次确认，补丁下载验证可使用" \ + "[CVE补丁工具](https://gitee.com/openeuler/cve-manager/blob/master/cve-agency-manager/cve_tracking/README.md)。\n" \ + "若补丁不准确，烦请在此issue下评论 '/report-patch 参考网址 补丁链接1,补丁链接2' 反馈正确信息，便于我们不断优化工具，不胜感激。\n" \ + "如 /report-patch https://security-tracker.debian.org/tracker/CVE-2021-3997 https://github.com/systemd/systemd/commit/5b1cf7a9be37e20133c0208005274ce4a5b5c6a1" + break + + body = table + instruction + await gitee.create_issue_comment(number=number, body=body) diff --git a/cve-agency-manager/cve_tracking/core/crawler/__init__.py b/cve-agency-manager/cve_tracking/core/crawler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e26da5218904e6406ea988ff828312ffb3f9fe2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/crawler/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/cve-agency-manager/cve_tracking/core/crawler/patch.py b/cve-agency-manager/cve_tracking/core/crawler/patch.py new file mode 100644 index 0000000000000000000000000000000000000000..97c44ffc27da8c19f73360112a68e3b685eb96c3 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/crawler/patch.py @@ -0,0 +1,297 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import re +import asyncio +from collections import namedtuple + +from constant import Constant +from core.platform import CvePlatform +from logger import logger +from util.github_api import Github +from util.gitlab_api import Gitlab +from conf import settings + + +class Patch: + """ + Patch acquisition, analysis implementation class + """ + + def __init__(self, cve_num, rpm_name): + self.cve_num = cve_num + self.rpm_name = rpm_name + self.patch_info_list = list() + self.patch_detail_list = list() + self.issue_pr_dict = dict() + self.pr_status_dict = dict() + self.github_api = Github() + self.gitlab_api = Gitlab() + self._SummaryInfo = namedtuple("SummaryInfo", ["pr_list", "issues_list"]) + self._PatchDetail = namedtuple("PatchDetail", ["platform", "details"]) + + @property + def issue_relation_info(self): + return {"issue": {"url": None, "prs": []}} + + @property + def pr_relation_info(self): + return {"url": None, "status": None, "commits": []} + + async def find_patches_detail(self): + """ + Further analyze the obtained patch information to get the final patch link + :return: self.patch_detail_list + """ + await self._find_patches_info() + # commit,pr,issue summary structure,convenient for deduplication and concurrent search + summary_info = self._SummaryInfo(pr_list=list(), issues_list=list()) + for patch_info in self.patch_info_list: + summary_info.pr_list.extend(patch_info.get("pr", [])) + summary_info.issues_list.extend(patch_info.get("issue", [])) + + # Find issue linked pr and add it to the pr list obtained earlier + if summary_info.issues_list: + issue_task_list = await self._create_async_task( + self._get_issue_link_pr, summary_info.issues_list + ) + issue_done_task, _ = await asyncio.wait(issue_task_list) + for task in issue_done_task: + summary_info.pr_list.extend(task.result()) + + if summary_info.pr_list: + # Query pr status + pr_status_task_list = await self._create_async_task( + self._get_pr_status, summary_info.pr_list + ) + await asyncio.wait(pr_status_task_list) + # Find pr contain commits + pr_task_list = await self._create_async_task( + self._get_pr_contain_commits, summary_info.pr_list + ) + await asyncio.wait(pr_task_list) + + # Generate the final information that needs to be output + self._convert_path_detail() + logger.info( + f'Find the patch information of cve "{self.cve_num}" as: \n{self.patch_detail_list}' + ) + return self.patch_detail_list + + async def _find_patches_info(self): + """ + Implementation method of preliminary search for CVE patch information + :return: self.patch_info_list + """ + logger.info( + f"Start to obtain patches info of {self.cve_num} for {self.rpm_name}" + ) + crawler_task = [ + asyncio.create_task( + CvePlatform( + cve_platform, + self.cve_num, + cve_platform["url"], + cve_platform["format"], + ).crawling_patch() + ) + for cve_platform in settings.get_platform() + ] + done_task, _ = await asyncio.wait(crawler_task) + self.patch_info_list.extend( + [task.result() for task in done_task if task.result()] + ) + + @staticmethod + async def _create_async_task(method, issue_pr_list): + """ + Create asynchronous tasks + :param method: Method to be executed + :param issue_pr_list: issue or pr list + :return: task list + """ + task_list = [] + for issue_or_pr in list(set(issue_pr_list)): + task_list.append(asyncio.create_task(method(issue_or_pr))) + return task_list + + async def _get_issue_link_pr(self, issue): + """ + Get the pr associated with the issue + :param issue: issue url + :return: pr url + """ + pr_list = [] + if Constant.GITHUB in issue: + pr_list = await self.github_api.issue_relevance_pull(issue_url=issue) + elif Constant.GITLAB in issue: + try: + issue_info = issue.split("/") + self.gitlab_api.set_attr(owner=issue_info[-5], repo=issue_info[-4]) + self._set_gitlab_host(issue) + issue_num = issue_info[-1] + response = await self.gitlab_api.issue_relevance_pull( + issue_id=issue_num + ) + pr_list = self._convert_api_response(response, url_key="web_url") + except IndexError: + logger.warning( + f"The issue link: {issue} does not conform to the standard format" + ) + else: + logger.warning(f"Issue {issue} failed to match the relevant code platform") + + # Record the relationship between issue and pr to facilitate subsequent patch information processing + logger.info(f"According to issue {issue} get pr: {str(pr_list)}") + + self.issue_pr_dict[issue] = pr_list + return pr_list + + async def _get_pr_status(self, pr): + """ + Query pr status + :param pr: pr + :return: self.pr_status_dict + """ + pr_number = self._set_owner_repo_by_pr(pr) + if not pr_number: + logger.info("This is not effective pr:%s" % pr) + return + if Constant.GITHUB in pr: + response = await self.github_api.get_pull_request(pr_number) + if response: + self.pr_status_dict[pr] = response.get("state") + + elif Constant.GITLAB in pr: + self._set_gitlab_host(pr) + response = await self.gitlab_api.get_single_mr(pr_number) + if response: + self.pr_status_dict[pr] = response.get("merge_status") + + async def _get_pr_contain_commits(self, pr): + """ + Get the commits contained in pr and record them + :param pr: pull requests + :return: self.issue_pr_dict + """ + pr_number = self._set_owner_repo_by_pr(pr) + if pr_number: + commits_list = [] + if Constant.GITHUB in pr: + response = await self.github_api.get_pull_commits(pr_number) + commits_list = self._convert_api_response(response, "html_url") + elif Constant.GITLAB in pr: + self._set_gitlab_host(pr) + response = await self.gitlab_api.get_mr_context_commits( + merge_request_iid=pr_number + ) + commits_list = self._convert_api_response(response, "web_url") + + logger.info(f"According to pull {pr} get commits: {str(commits_list)}") + + self.issue_pr_dict[pr] = commits_list + + def _set_owner_repo_by_pr(self, pr): + """ + Set the owner and repo required by the api through the pr link + :param pr: pull request + :return: pr number + """ + pr_info = pr.split("/") + try: + pr_number = pr_info[-1] + if Constant.GITHUB in pr: + self.github_api.set_attr(owner=pr_info[-4], repo=pr_info[-3]) + elif Constant.GITLAB in pr: + self.gitlab_api.set_attr(owner=pr_info[-5], repo=pr_info[-4]) + else: + logger.warning(f"Pr {pr} failed to match the relevant code platform") + return None + except IndexError: + logger.warning( + f"The pull link: {pr} does not conform to the standard format" + ) + return None + + return pr_number + + @staticmethod + def _convert_api_response(response, url_key): + """ + Process api return result, get pull url or commit url + :param response: api response + :param url_key: url key + :return: pull list or commits list + """ + pr_commits_list = list() + if response: + pr_commits_list.extend( + [commit_info.get(url_key) for commit_info in response] + ) + return pr_commits_list + + def _set_gitlab_host(self, issue_pr): + """ + Set the host of gitlab + :param issue_pr: issue or pull + :return: None + """ + _host_match = re.search(pattern=Constant.GITLAB_HOST_REGEX, string=issue_pr) + _host = _host_match.group() + self.gitlab_api.host = _host + + def _set_pr_relation_info(self, pr): + pr_relation_info = self.pr_relation_info + pr_relation_info["url"] = pr + pr_relation_info["commits"] = self.issue_pr_dict.get(pr, []) + pr_relation_info["status"] = self.pr_status_dict.get(pr) + return pr_relation_info + + def _convert_path_detail(self): + """ + Process data and generate patch details + :return: self.patch_detail_list + """ + for patch_info in self.patch_info_list: + patch_detail = self._PatchDetail( + platform=patch_info.get("platform"), details=list() + ) + issue_relation_info = self.issue_relation_info + + # First, If there is commits information, pr and issue information will be ignored + if patch_info.get("commits"): + pr_relation_info = self.pr_relation_info + pr_relation_info["commits"] = patch_info.get("commits") + issue_relation_info["issue"]["prs"].append(pr_relation_info) + + # Second, processing pr information + for pr in patch_info.get("pr"): + issue_relation_info["issue"]["prs"].append( + self._set_pr_relation_info(pr) + ) + + # None Issue associated patch + if issue_relation_info["issue"]["prs"]: + patch_detail.details.append(issue_relation_info) + + # Third, processing issue information + for issue in patch_info.get("issue"): + issue_relation_info = self.issue_relation_info + issue_relation_info["issue"]["url"] = issue + for pr in self.issue_pr_dict.get(issue, []): + issue_relation_info["issue"]["prs"].append( + self._set_pr_relation_info(pr) + ) + + patch_detail.details.append(issue_relation_info) + + self.patch_detail_list.append(dict(patch_detail._asdict())) diff --git a/cve-agency-manager/cve_tracking/core/download/__init__.py b/cve-agency-manager/cve_tracking/core/download/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e26da5218904e6406ea988ff828312ffb3f9fe2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/download/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/cve-agency-manager/cve_tracking/core/download/save.py b/cve-agency-manager/cve_tracking/core/download/save.py new file mode 100644 index 0000000000000000000000000000000000000000..329b4cec08fd874f34895101fe9ab342ec0b7e10 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/download/save.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import os +import re +from conf import CONFIG, settings +from logger import logger +from request import http + + +def _file(cve, file): + """ + save file path + + :param cve: fiex cve number + :param file: save path + """ + file = os.path.join(file, cve) + os.makedirs(file, exist_ok=True) + file_name = ( + "fix-cve-" + + str(sum([os.path.isfile(list_index) for list_index in os.listdir(file)]) + 1) + + ".patch" + ) + return os.path.join(file, file_name) + + +def _patch_url(url): + download_rules = settings.configuration.get("download") + if not download_rules: + return url + ".patch" + + for replace in filter(lambda x: x.get("action") == "REPLACE", download_rules): + if re.findall(replace.get("regex", ""), url): + target_regex = replace.get("target", ["commit", "patch"]) + return re.sub(target_regex[0], target_regex[-1], url) + + for append in filter(lambda x: x.get("action") == "APPEND", download_rules): + if re.findall(append.get("regex", ""), url): + target_regex = append.get("target", [".patch"]) + return url + target_regex[-1] + + +def save_patch(patch_details, cve, path=CONFIG.PATCH_SAVE_PATH): + """ + Download and save the patch file + :param patch_details: patch info + :param cve: cve number + :param path: patch path + :return: None + """ + commits = set() + for patch in patch_details: + for issue_info in patch.get("details", list()): + for pr in issue_info["issue"].get("prs", list()): + commits.update(pr.get("commits", [])) + + # download patch + for url in commits: + out_file = _file(cve=cve, file=path) + patch_url = _patch_url(url=url) or url + file = http.download(url=patch_url, out_fname=out_file) + if not file: + logger.error("Failed to download the file: %s" % patch_url) + + +__all__ = ("save_patch",) + diff --git a/cve-agency-manager/cve_tracking/core/feedback/__init__.py b/cve-agency-manager/cve_tracking/core/feedback/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e26da5218904e6406ea988ff828312ffb3f9fe2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/feedback/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py b/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..ec2250d065d5082eb85ba97e0f8e9299f450a669 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from conf import CONFIG +from logger import logger +from util.gitee_api import Gitee +from util.github_api import Github +from util.gitlab_api import Gitlab + + +class IssueMode: + """ + Feedback by creating an issue + """ + + def __init__(self, cve_num, cve_platform, patch_url, issue_platform): + self.cve_num = cve_num + self.cve_platform = cve_platform + self.patch_url = patch_url + self.issue_platform = issue_platform + self._support_issue_platform = {'gitee': Gitee, + 'github': Github, + 'gitlab': Gitlab} + + async def create_feedback_issue(self): + """ + Create an issue about tool feedback + :return: True or False + """ + platform_api_class = self._support_issue_platform.get(self.issue_platform) + if platform_api_class is None: + logger.error(f'Issue hosting platform: {self.issue_platform} does not support') + return False + if not all([CONFIG.FEEDBACK_ISSUE_OWNER, CONFIG.FEEDBACK_ISSUE_REPO]): + logger.error(f'"FEEDBACK_ISSUE_OWNER" or "FEEDBACK_ISSUE_REPO" no setting, Please set in file "config.ini"') + return False + + title = f"CVE: {self.cve_num} patch information" + description = f"Cve reference URL: {self.cve_platform}\nPatch url: {self.patch_url}" + platform_api = platform_api_class() + platform_api.set_attr(CONFIG.FEEDBACK_ISSUE_OWNER, CONFIG.FEEDBACK_ISSUE_REPO) + response = await platform_api.create_issue(title=title, body=description) + return response is not None diff --git a/cve-agency-manager/cve_tracking/core/platform/__init__.py b/cve-agency-manager/cve_tracking/core/platform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7d26770ced7f7dbc9adb8fa2c600ecfc8c44d04 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/platform/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from .cve_platform import CvePlatform + +__all__ = ("CvePlatform",) diff --git a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py new file mode 100644 index 0000000000000000000000000000000000000000..a057edf0c56d225d74f38b5097b62c02b0064571 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py @@ -0,0 +1,198 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import json +from json import JSONDecodeError +import re +from collections import namedtuple +from functools import wraps + +from bs4 import BeautifulSoup + +from constant import Constant +from exception import RequestError +from logger import logger +from request import http +from conf import settings + + +def match(regexes, component): + """ + Decorator match commit, pr, issue + :param regexes: regexes list + :param component: commit, pr, issue + :return: func + """ + + def _inner_decorator(func): + @wraps(func) + def _inner_match(crawler_text, patch_info): + if patch_info.get("commits") or patch_info.get("pr"): + return func(crawler_text, patch_info) + match_list = list() + for regex in regexes: + match_list.extend(re.findall(pattern=regex, string=crawler_text)) + try: + patch_info[component].extend(list(set(match_list))) + except KeyError: + logger.error(f"Patch info does not exist in field {component}") + return func(crawler_text, patch_info) + + return _inner_match + + return _inner_decorator + + +class CvePlatform: + """ + The parent class of cve reference URL + """ + + def __init__( + self, + platform, + cve_num=None, + base_url=None, + format_text="text", + ): + self._platform = platform + self.cve_num = cve_num + self.base_url = base_url + self._format = format_text + self._Patch = namedtuple("Patch", ["platform", "commits", "pr", "issue"]) + + @property + def crawler_url(self): + """ + Reference URL where cve information is located + :return: url + """ + return self.base_url.format(cve_num=self.cve_num) + + @property + def patch_info(self): + """ + Structure of cve patch information + :return: instance of _Patch + """ + return self._Patch(platform=self.crawler_url, commits=[], pr=[], issue=[]) + + async def _rule_redirect(self, response): + """ + Page multi layer jump data parsing + :param response: http response data + :return: response data + """ + for redirect_rule in self._platform.get("redirect", []): + format_text = self.format_text(response.text) + target_val = list( + set( + re.findall( + pattern=redirect_rule.get("regex", ""), string=format_text or "" + ) + ) + ) + try: + url = redirect_rule["prefix"] + target_val[-1] + except IndexError: + url = redirect_rule["prefix"] + response = await self._method(redirect_rule)( + url, data=redirect_rule.get("body") + ) + + return response + + @staticmethod + def _method(rule): + method = http.get if rule.get("method", "get") == "get" else http.post + return method + + async def crawling_patch(self): + """ + Crawl patch information from the cve reference website + :return: patch info + """ + try: + _response = await self._method(self._platform)( + self.crawler_url, data=self._platform.get("body") + ) + if "redirect" in self._platform: + _response = await self._rule_redirect(response=_response) + except RequestError: + return None + + if _response.error or not _response.text: + logger.error( + f"Failed to access URL {self.crawler_url}, detail: {_response.error}" + ) + return None + + formatted_text = ( + self.format_text(_response.text) + if self._format == "text" + else self.json(_response.text) + ) + if formatted_text is None: + logger.error(f"Failed to format string {_response.text}") + return None + + patch_info_dict = self.match_patch( + formatted_text, dict(self.patch_info._asdict()) + ) + return patch_info_dict + + @staticmethod + @match(settings.get_regex(label="commit"), "commits") + @match(settings.get_regex(label="pr"), "pr") + @match(settings.get_regex(label="issue"), "issue") + def match_patch(text, patch_info): + """ + Matching patch related links,Use decorator to find + :param text: Crawled content + :param patch_info: self.patch_info + :return: None + """ + # Matching url de-duplication + for key, value in patch_info.items(): + if isinstance(value, list): + patch_info[key] = list(set(value)) + + logger.info(f"Find patch: {patch_info}") + return patch_info + + @staticmethod + def format_text(text): + """ + Format the content obtained from the URL + :return: Formatted content + """ + if not text: + return None + html_text = BeautifulSoup(text, "html.parser") + return html_text.prettify() + + @staticmethod + def json(text): + """ + Rewrite the formatted web page to get the content method + :param text: content of web page + :return: formatted content + """ + try: + text_dict = json.loads(text) + return json.dumps(text_dict, indent=4) + except JSONDecodeError as e: + logger.error( + f"The format of the content obtained by bugzilla website is incorrect, " + f"content is {text}, message is {e.msg}" + ) + return None diff --git a/cve-agency-manager/cve_tracking/core/verification/__init__.py b/cve-agency-manager/cve_tracking/core/verification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e26da5218904e6406ea988ff828312ffb3f9fe2 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/verification/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/cve-agency-manager/cve_tracking/core/verification/apply.py b/cve-agency-manager/cve_tracking/core/verification/apply.py new file mode 100644 index 0000000000000000000000000000000000000000..6cf5b533c147d9a2893c33f686542492f8a09376 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/verification/apply.py @@ -0,0 +1,86 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import os.path +import stat +import subprocess + +from exception import ConfigNotFoundError, PathNotExistError, PathEmptyError +from logger import logger + +CURRENT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__))) + + +class PathApply: + """ + Patch verification class + """ + + def __init__(self, rpm_name, patch_path, source_path, branch_rpm): + self.rpm_name = rpm_name + self.patch_path = patch_path + self.source_path = source_path + self.branch_rpm = branch_rpm + + def packing_source(self): + """ + Package verification implementation class + :return: None + """ + logger.info( + f'Start to apply patch in "{self.patch_path}" of package "{self.rpm_name}", branch is "{self.branch_rpm}"') + self._validate_path() + self._change_shell_right() + run_shell = os.path.join(CURRENT_PATH, 'run.sh') + try: + output = subprocess.check_output( + [run_shell, self.rpm_name, self.branch_rpm, self.patch_path, self.source_path], + stderr=subprocess.STDOUT, + shell=False) + apply_result = output.decode('utf-8') + logger.info(f'Patch verification process:\n{apply_result}') + logger.info(f'Apply patch successfully, log is {self.source_path}/{self.rpm_name}/result.log') + except subprocess.CalledProcessError as e: + logger.info(f'Patch verification process:\n{e.output.decode("utf-8")}') + logger.error(f'Apply patch failed, log is {self.source_path}/{self.rpm_name}/result.log') + + def _validate_path(self): + """ + Verify the legitimacy of each path + :return: None + :raise: ConfigNotFoundError PathNotExistError PathEmptyError + """ + if not self.patch_path or not self.source_path: + raise ConfigNotFoundError('Patch path (PATCH_SAVE_PATH) or source path (SOURCE_CODE_PATH) does not exist') + + if not os.path.isdir(self.source_path): + logger.warning(f'The source package download path: {self.source_path} does not exist, it will be created') + try: + os.makedirs(self.source_path) + except PermissionError: + logger.error(f'Insufficient permissions to create a path') + raise PathNotExistError(self.source_path) + if not os.path.isdir(self.patch_path): + raise PathNotExistError(self.patch_path) + + if not os.listdir(self.patch_path): + raise PathEmptyError(self.patch_path) + + @staticmethod + def _change_shell_right(): + """ + Give shell script execution permissions + :return: None + """ + for file in os.listdir(CURRENT_PATH): + if str(file).endswith('.sh'): + os.chmod(os.path.join(CURRENT_PATH, file), 0o0755) diff --git a/cve-agency-manager/cve_tracking/core/verification/common.sh b/cve-agency-manager/cve_tracking/core/verification/common.sh new file mode 100644 index 0000000000000000000000000000000000000000..cfb1614def3b01422173a8fbde8ccd8aff5b3ceb --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/verification/common.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +function pkg_manager() { + # Judge the software package management mechanism by OS version + os_version=$(cat /etc/os-release) + if [[ ${os_version} =~ "openEuler" ]] || [[ ${os_version} =~ "Red Hat" ]] || [[ ${os_version} =~ "CentOS" ]]; then + return 0 + elif [[ ${os_version} =~ "Ubuntu" ]] || [[ ${os_version} =~ "Debian" ]]; then + return 1 + else + echo "[ERROR] Linux distributions not support" + exit 1 + fi +} + +function install_rpm() { + yum_install_rpm_name=$1 + apt_install_rpm_name=$2 + # Check if rpm is installed + os_version=$(pkg_manager) + if [[ $? -eq 0 ]]; then + install_rpm_name=${yum_install_rpm_name} + install_status=$(rpm -qa "${install_rpm_name}" 2>&1) + else + install_rpm_name=${apt_install_rpm_name} + install_status=$(dpkg -s "${install_rpm_name}" 2>&1) + fi + + if [[ -z ${install_status} ]] || [[ ${install_status} =~ "is not installed" ]] || [[ ${install_status} =~ "没有安装" ]]; then + echo "[INFO] ${install_rpm_name} is not installed, start install it" + sudo apt install "${install_rpm_name}" -y >/dev/null 2>&1 + apt_install=$? + sudo yum install "${install_rpm_name}" -y >/dev/null 2>&1 + yum_install=$? + if [[ ${apt_install} -eq 1 ]] && [[ ${yum_install} -eq 1 ]]; then + echo "[ERROR] ${install_rpm_name} install failed" + exit 1 + fi + fi +} diff --git a/cve-agency-manager/cve_tracking/core/verification/packing.sh b/cve-agency-manager/cve_tracking/core/verification/packing.sh new file mode 100644 index 0000000000000000000000000000000000000000..8ab7db464c3137f4c8dafd12c1bc3ddbdf2ed956 --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/verification/packing.sh @@ -0,0 +1,179 @@ +#!/bin/bash +# shellcheck disable=SC2068 +# shellcheck disable=SC2164 +# shellcheck disable=SC2010 +# shellcheck disable=SC2063 +# shellcheck disable=SC2062 +# shellcheck disable=SC2035 + +rpm_name=$1 +rpm_branch=$2 +patches_path=$3 +source_path=$4 +path_file_name="" +install_retry_count=0 +# The rpm package that needs to be installed when compiling +requires_rpms="" +spec_file="" +home_path=$( + cd ~ + pwd -P +) +root_build_path=${home_path}/"rpmbuild" +current_path=$( + cd "$(dirname "$0")" + pwd +) +. ${current_path}/common.sh + +function check_patch() { + if [[ -z ${patches_path} ]]; then + echo "[ERROR] Input patch is null" + exit 1 + fi + + for patch in ${patches_path[@]}; do + if [[ ! -f ${patch} ]]; then + echo "[ERROR] ${patch} is not found" + exit 1 + fi + done +} + +function check_local() { + if [[ ! -d ${source_path} ]]; then + echo "[ERROR] The source package path ${source_path} does not exist" + exit 1 + fi +} + +function check_remote() { + if [[ -z ${rpm_name} ]] || [[ -z ${rpm_branch} ]]; then + echo "[ERROR] Incorrect input,Please use [ /bin/bash add_patch.sh 'name of package' 'branch of package' 'path of patch file']" + exit 1 + fi +} + +function pre_env_build() { + echo "[INFO] Create root path of rpmbuild" + if [[ ! -d ${root_build_path} ]]; then + mkdir -p ${root_build_path}/{BUILD,BUILDROOT,RPMS,SPECS,SOURCES,SRPMS} + fi +} + +function git_clone() { + echo "[INFO] Start to git clone ${rpm_name}" + cd ${source_path} + git clone -b ${rpm_branch} https://gitee.com/src-openeuler/${rpm_name}.git >/dev/null 2>&1 + rpm_path=$(ls | grep ${rpm_name}) + if [[ -z ${rpm_path} ]]; then + echo "[ERROR] Access https://gitee.com/src-openeuler/${rpm_name}.git clone ${rpm_name} failed, please check path ${source_path}" + exit 1 + fi + + source_path=${source_path}/${rpm_name} +} + +function update_spec() { + echo "[INFO] Start to update spec file" + cd ${source_path} + spec_file=$(ls | grep *.spec | grep -v *.bak | grep -v *.update) + if [[ -z ${spec_file} ]]; then + echo "[ERROR] spec file is not found" + exit 1 + fi + # backup spec + /bin/cp ${spec_file} ${spec_file}.bak + # update Release + release_version=$(grep "Release:" ${spec_file} | awk -F " " '{print $NF}' | tr -cd "[0-9]") + new_release=$(expr ${release_version} + 1) + sed -i "s/Release:.*${release_version}/Release: ${new_release}/" ${spec_file} + # add Patch*** + last_patch=$(grep "Patch.*:" ${spec_file} | sed -n '$p') + if [[ -z ${last_patch} ]]; then + source_row=$(grep -n "Source.*:" ${spec_file} | sed -n '$p' | awk -F ':' '{print $1}') + sed -ie "${source_row}G;${source_row}a Patch0000: ${path_file_name}" ${spec_file} + else + last_patch_row=$(grep -n "${last_patch}" ${spec_file} | awk -F ':' '{print $1}') + last_patch_num=$(echo ${last_patch} | awk -F ':' '{print $1}' | awk -F 'Patch' '{print $2}') + patch_name_len=${#last_patch_num} + new_patch_num=$(expr ${last_patch_num} + 1) + new_patch_num=$(printf "%0${patch_name_len}d" ${new_patch_num}) + sed -i "${last_patch_row}a Patch${new_patch_num}: ${path_file_name}" ${spec_file} + fi + # add %patch + last_patch_apply=$(grep "%patch.* " ${spec_file} | sed -n '$p') + if [[ -n ${last_patch_apply} ]]; then + last_patch_apply_row=$(grep -n "${last_patch_apply}" ${spec_file} | awk -F ':' '{print $1}') + last_patch_apply_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $1}' | awk -F 'patch' '{print $2}') + ignore_level_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $2}') + new_patch_apply_num=$(expr ${last_patch_apply_num} + 1) + sed -i "${last_patch_apply_row}a %patch${new_patch_apply_num} ${ignore_level_num} " ${spec_file} + fi + # add changelog + change_log_row=$(grep -n '%changelog' ${spec_file} | sed -n '$p' | awk -F ':' '{print $1}') + date_now=$(date '+%a %b %d %Y') + version=$(grep 'Version:' ${spec_file} | awk -F ' ' '{print $NF}') + log_description="- add ${path_file_name}" + log_title="* ${date_now} robot - ${version}-${new_release}" + sed -i "${change_log_row}G" ${spec_file} + sed -i "${change_log_row}a ${log_description}" ${spec_file} + sed -i "${change_log_row}a ${log_title}" ${spec_file} + + echo "[INFO] Update spec file success" +} + +function mv_source_file() { + echo "[INFO] Copy source file to ${root_build_path}" + cd ${source_path} + spec_file=$(ls | grep *.spec | grep -v *.bak | grep -v *.update) + /bin/cp ${patches_path} ${source_path} + /bin/cp * ${root_build_path}/SOURCES + /bin/cp *.spec ${root_build_path}/SPECS +} + +function rpm_build() { + echo "[INFO] Start to rpmbuild" + install_rpm rpm-build rpm + apt-get build-dep ${rpm_name} >/dev/null 2>&1 + dnf builddep ${root_build_path}/SPECS/${spec_file} >/dev/null 2>&1 + rpmbuild -bp ${root_build_path}/SPECS/${spec_file} >./result.log 2>&1 + if [[ $? -eq 0 ]]; then + echo "[INFO] build success !!!" + exit 0 + elif [[ -n $(grep "Failed build dependencies" ./result.log) ]] && [[ ${install_retry_count} -le 2 ]]; then + requires_rpms=$(grep -r "is needed by" ./result.log | awk -F " " '{print $1}') + echo "${requires_rpms}" >./requires_rpms.log + for rpm in ${requires_rpms[@]}; do + install_rpm ${rpm} ${rpm} + done + install_retry_count=$(expr ${install_retry_count} + 1) + rpm_build + else + mv ${source_path}/${spec_file} ${source_path}/${spec_file}.update + mv ${source_path}/${spec_file}.bak ${source_path}/${spec_file} + echo "[ERROR] build failed,log is ${source_path}/result.log" + exit 1 + fi +} + +function main() { + check_patch + if [[ ${rpm_branch} == "no" ]]; then + check_local + pre_env_build + else + check_remote + pre_env_build + git_clone + fi + for patch in ${patches_path[@]}; do + path_file_name=$(echo "${patch}" | awk -F "/" '{print $NF}' | awk -F "\\" '{print $NF}') + update_spec + done + + mv_source_file + rpm_build +} + +main diff --git a/cve-agency-manager/cve_tracking/core/verification/run.sh b/cve-agency-manager/cve_tracking/core/verification/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..9b9f073edda7b76eed86b0f1e610949001b6ed0f --- /dev/null +++ b/cve-agency-manager/cve_tracking/core/verification/run.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# shellcheck disable=SC2068 +# shellcheck disable=SC2164 +# shellcheck disable=SC2002 +# shellcheck disable=SC2045 +# shellcheck disable=SC2115 +rpm_name=$1 +remote_branch=$2 +patch_path=$3 +source_path=$4 +current_path=$( + cd "$(dirname "$0")" + pwd +) +. ${current_path}/common.sh + +function check() { + # Check input legitimacy + if [[ -z ${rpm_name} ]] || [[ -z ${remote_branch} ]]; then + echo "[ERROR] rpm name or gitee branch not specified" + exit 1 + fi + # Check if the path exists + if [[ ! -d ${patch_path} ]] || [[ ! -d ${source_path} ]]; then + echo "[ERROR] The patch file directory: ${patch_path} or the source code download directory: ${source_path} does not exist is not exist" + exit 1 + fi +} + +function run() { + patch_files=$(ls ${patch_path}) + patch_ids="" + new_patches="" + for patch_file in ${patch_files[@]}; do + patch_file=${patch_path}/${patch_file} + git_patch_ids=$(cat "${patch_file}" | git patch-id --stable) + if [[ ! ${patch_ids} =~ ${git_patch_ids} ]]; then + new_patches="${new_patches} ${patch_file}" + patch_ids="${patch_ids} ${git_patch_ids}" + else + echo "[INFO] ${patch_file} duplicate content" + fi + done + # Perform code download, modify spec file, package verification operation + /bin/bash ${current_path}/packing.sh "${rpm_name}" "${remote_branch}" "${new_patches}" "${source_path}" + add_patch_result=$? + if [[ ${add_patch_result} -eq 0 ]]; then + echo "[INFO] rpm: \"${rpm_name}\" of branch: \"${remote_branch}\" apply patch \"${new_patches}\" successfully" + exit 0 + else + echo "[ERROR] rpm: \"${rpm_name}\" of branch: \"${remote_branch}\" failed to apply patch \"${new_patches}\"" + exit 1 + fi +} + +function main() { + # Executive total entrance + check + install_rpm git git + run +} + +main diff --git a/cve-agency-manager/cve_tracking/cve-tracking.yaml b/cve-agency-manager/cve_tracking/cve-tracking.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e2873990201c4c190e3de1ec29ab7ba0f0b5660 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cve-tracking.yaml @@ -0,0 +1,112 @@ +# Regular expression to match cve patch info +# 匹配cve补丁信息的正则表达式 +regex: + # A tag that matches the type of content, currently only "commit" "pr" "issue" is supported + # 匹配内容的类型的标签，当前只支持 "commit"/"pr"/"issue" + - label: commit + # Regular expressions that match specific types of content, you can set multiple matching rules + # 匹配特定类型内容的正则表达式，可以设置多个匹配规则 + regular: + - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40} + - http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40} + - label: pr + regular: + - http[s]?://(?:[-\w.\/;?])+(?:pull[s]?|merge_requests)/[1-9][0-9]* + - label: issue + regular: + - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+ + +# Platforms to find bug fixes for cve, such as Debian, Bugzilla, Nvd, etc. +# 查找cve漏洞修复的平台，例如 Debian、Bugzilla、Nvd等 +platform: + # Platform name (required) + # 平台的名称(必配项) + - name: Cnnvd + # The request address for finding cve details, "{cve num}" is a placeholder for string replacement in python (required) + # 查找cve详情信息的请求地址，“{cve_num}”是python中字符串替换的占位符(必配项) + url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num} + # The method of sending the request, the default is get request, and it can also be specified as post request. + # When the request method is get, it can not be configured here. + # 发送请求的方式，默认为get请求，还可以指定为post请求，当请求方式为get时，此处可不配置 + method: get + # If the request method is post and there is a request body, the body is required and must be in json format. + # When the request method is get, it can be omitted here. + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式，当请求方式为get时，此处可不配置 + body: + # The way to parse the response body, currently only "text" or "json" is supported (required) + # 解析响应体的方式，当前只支持“text”或“json”(必配项) + format: text + # When the request cannot directly obtain the cve information (multiple jumps according to the specific link in + # the page), you can specify the method of page jump to obtain further information + # 当请求无法直接获取cve信息时（根据页面中特定链接多次跳转），可以指定页面跳转的方式来获取进一步的信息 + redirect: + # The prefix of the jump address, generally the domain name of the jump page, if the jump address matched by + # the regular expression has a complete domain name + path, this configuration item can be left blank + # 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写 + - prefix: http://cnnvd.org.cn + # The regular expression that matches the page jump address, if it matches multiple values, get the last match + # 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项 + regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+ + # The method of sending the request, the default is get request, and it can also be specified as post request. + # When the request is get, it can not be configured here. + # 送请求的方法，默认为get请求，还可以指定为post请求，当请求为get时，此处可不配置 + method: get + # If the request method is post and there is a request body, the body is required and must be in json format + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式 + body: + + - name: Debian + url: https://security-tracker.debian.org/tracker/{cve_num} + format: text + + - name: Ubuntn + url: https://ubuntu.com/security/{cve_num} + format: text + + - name: Bugzilla + url: https://bugzilla.redhat.com/rest/bug/{cve_num}/comment + format: json + + - name: Nvd + url: https://nvd.nist.gov/vuln/detail/{cve_num} + format: text + + - name: Suse + url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num} + format: text + +# Private token for api access, currently only supports "gitee" "github" "gitlab" +# api访问时的私人令牌，当前只支持"gitee"/"github"/"gitlab" +authentication: + # api name, this value cannot be changed + # api名称，此值不可更改 + - name: gitee + # A private token for accessing the api, which can be manually obtained and changed later + # 访问api的私人令牌，可以手动获取后更改 + token: + - name: github + token: + - name: gitlab + token: glpat-24DGxkuTA8nTR1YQsXvZ + +# 补丁包下载配置项 +download: + # 补丁包下载路径的调整，分为（REPLACE、APPEND）两种方式，分别对应了补丁路径的替换和拼接 + - action: REPLACE + # 匹配url的正则表达式，符合特定规则的url进行特定值的替换（必填项） + regex: http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40} + # 替换的规则，第一个值为需要替换的内容，第二个值为替换的目标内容，如果不传递默认为（commit,patch）进行替换操作（可选配置项） + target: + - commit + - patch + - action: REPLACE + regex: http[s]?://(?:[-\w.\/;?])+/commit[s]?/\?id=[0-9a-z]{8,40} + target: + - commit + - patch + - action: APPEND + # 匹配url的正则表达式，符合特定规则的url进行特定值的拼接（必填项） + regex: http[s]?://(?:[-\w.\/;?])+/commit[s]?/[0-9a-z]{8,40} + # 拼接的规则，在url地址的末尾需要拼接的内容，如果不传递默认拼接“.patch”(可选配置项) + target: + - .patch diff --git "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000000000000000000000000000000000000..d34b266c4448a14b3b04fda53fd5b910ef8f8d5c --- /dev/null +++ "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,436 @@ +# CVE 补丁自动获取工具设计文档 + +## 一、需求概述 + +### 1.1、需求描述 + +开源社区中大部分软件都是第三方开源软件，这无法避免会存在漏洞，快速发现并及时修复 cve 漏洞是衡量一个开源社区是否健康的重要 指标之一。cve_tracking 工具旨在自动查找 cve 的修复补丁，减少人工查找时间，提高 cve 修复速度，保障社区健康运行。 + +### 1.2、受益人 + +| 角色 | 角色描述 | +| ------------ | ------------------------------------------------------------------------------------------------------------------- | +| 版本经理 | 负责版本发布、维护和进度看护的人员，版本发布要求解决高分 CVE，cve_tracking 可以帮助快速解决 CVE，保证版本发布进度。 | +| 社区维护人员 | 负责修复 CVE 的人员，可以使用 cve_tracking，减少查找工作量，提高解决速度。 | +| 社区开发人员 | 自研软件包的开发人员，可以使用补丁验证模块功能进行补丁验证。 | + +### 1.3、依赖组件 + +| 组件 | 组件描述 | 可获得性 | +| ------------------------------ | ---------------------------------------------- | ---------------- | +| python3 | Python 程序运行环境，需要 python3.6 以上版本。 | openEuler 已集成 | +| python3-retrying | requests 请求重试依赖程序库 | openEuler 已集成 | +| python3-concurrent-log-handler | python 日志程序库，包括日志记录，日志转储 | openEuler 已集成 | +| python3-pyyaml | python 处理 yaml 文件程序库 | openEuler 已集成 | +| python3-asyncio | python 异步执行 io 操作程序库 | openEuler 已集成 | +| python3-bs4 | python 处理 html 格式程序库 | openEuler 已集成 | +| python3-aiohttp | python 异步请求 web 资源操作程序库 | openEuler 已集成 | +| python3-wget | python 下载 web 资源程序库 | openEuler 已集成 | +| python3-fake_useragent | python 请求 api 设置代理程序库 | openEuler 已集成 | +| python3-lxml | python 解析 xml 程序库 | openEuler 已集成 | + +### 1.4、特性需求 + +| 需求编号 | 需求描述 | 特性描述 | +| -------- | ---------- | ---------------------------------------------------------------------------- | +| 1 | 补丁查找 | 通过 cve 参考网址和查找规则进行补丁链接匹配查找。 | +| 2 | issue 评论 | 将查找到的补丁链接添加到相关 issue 的评论中。 | +| 3 | 补丁下载 | 通过查找到的补丁链接，将补丁文件下载保存。 | +| 4 | 补丁验证 | 下载补丁文件后，进行补丁打包验证。 | +| 5 | 补丁反馈 | 若工具为找到补丁，通过命令向指定仓库提交 issue，方便数据统计和后续工具优化。 | + +## 二、设计概述 + +### 2.1、 设计原则 + +- 数据与代码分离： 功能实现是需要考虑哪些数据是需要配置动态改变的，不可在代码中将数据写死，应提取出来做成可配置项。 +- 接口与实现分离：外部依赖模块接口而不是依赖模块实现。 +- 模块划分： 不同功能划分不同模块，使用命令行参数进行灵活组合，模块之间保证无循环调用。 +- 并行处理：采用多线程，协程等技术，并行请求外部 api，减少 IO 等待时间，提高处理速度。 +- 变更： 架构设计变更、外部接口变更、特性需求变更需要结合开发、测试等所有相关人员经过开会讨论后决策，不可擅自变更。 + +## 三、需求分析 + +### 3.1、使用场景 + +- 在线使用：和 cve_manager 集成，提交 cve 的 issue 时自动触发，也可以通过/find-patch 命令进行触发，该方式只输出参考网址，相关 pr 和补丁链接，无下载和验证信息。 + +> 评论格式 +> +> | 参考网址 | 关联 pr | 状态 | 补丁链接 | +> | --------------------------------------------------------- | ---------------------------------------------- | ----- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +> | https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2020-1735 | https://github.com/ansible/ansible/pull/68780/ | close | https://github.com/ansible/ansible/pull/68780/commits/5292482553dc409081f7f4368398358cbf9f8672，https://github.com/ansible/ansible/pull/68780/commits/5292482553dc409081f7f4368398358cbf9f8673 | +> | https://ubuntu.com/security/CVE-2019-10156 | | | https://github.com/ansible/ansible/pull/57188/commits/8254c266f962d5febe46396d5083bb9c1da74840，https://github.com/ansible/ansible/pull/57188/commits/fbda0028750a17a032d83dad9d1fb284f9ea68a4 | + +​ 反馈功能：如果工具未找到补丁信息，但是人工可以找到，可以通过在评论中添加一下内容进行反馈： + +```shell +/feedback "cve参考网址" "补丁url" +``` + +​ 该操作将在https://gitee.com/liheavy/cve_tracking仓库中提交相关issue。可通过config.ini中的"FEEDBACK"中的配置来更改提交issue所在的仓库。 + +- 本地使用：下载工具后，通过 python 调用，该方式支持下载补丁和验证补丁的操作，也支持补丁反馈功能，具体操作见 readme 指导文档。 + +### 3.2、用例视图 + +![](images\用例图.png) + +> 说明： +> +> - 社区维护人员是该工具的主要使用者，用于快速查找补丁，及时修复 cve，当工具未找到相关补丁时可使用反馈功能进行反馈。 +> - 社区开发人员可以使用补丁验证模块，快速打包验证，可根据反馈信息不断提升工具的查找率和正确率。 +> - 版本经理主要是利用该工具，能更好保障版本的顺利发布。 + +### 3.3、逻辑视图 + +![](images\逻辑视图.png) + +### 3.4、代码结构 + +![](images\代码模块.png) + +### 3.5、时序视图 + +![](images\时序图.png) + +### 3.6、开发视图 + +![](images/开发视图.png) + +### 3.7、质量设计 + +##### 3.7.1、性能规格 + +| 规格项 | 功能点 | 规格指标 | +| ------ | ---------------------- | ----------------------------------------------------------------------------------------------------- | +| 效率 | 补丁下载 | 由于补丁平台的网络访问可能存在不稳定性以及补丁大小区别，将重试三次，每次重试 2min，总时间限制为 10min | +| 效率 | 提交评论
工具反馈 | 由于补丁平台的网络访问可能存在不稳定性，将重试三次，每次重试时间为 1min，总时间限制为 3min | +| 效率 | 补丁验证 | 由软件包编译流程和编译环境决定，无时间限制。 | + +**补丁查找性能提升策略：** + +![](images\性能策略.png) + +##### 3.7.2、可靠性设计 + +| 影响因素 | 使用场景 | 可靠性说明 | +| ---------- | :--------- | ------------------------------------------------------------------------------------------------------------------------------- | +| 高并发 | 在线、本地 | 基于当前主要使用场景为在线使用，每次执行只输入单个 cve，所以多个请求也只会顺序执行，资源控制由操作系统调度。 | +| 进程异常 | 在线、本地 | 当前工具为 python 调用方式，当系统故障或者进程异常退出，执行中断，后续执行不会受影响。 | +| 结果正确性 | 在线、本地 | 工具采用多个 cve 参考平台共同查找的方式，平台信息各自展示，可通过交叉对比提升结果准确性。同时提供反馈功能，不断提升工具准确性。 | + +##### 3.7.3、安全性设计 + +| 功能点 | 涉及安全项 | 解决策略 | +| ---------------- | ---------------- | ------------------------------------------------------------------------------------------- | +| 上游社区信息获取 | 上游社区 token | 不需要登录的将直接访问，必须登录的将支持用户在配置文件配置临时 token。 | +| issue 评论 | gitee 私人令牌 | 主要使用场景为和 cve_manager 集成，将使用机器人令牌。本地执行支持用户配置临时全局环境变量。 | +| 补丁验证 | 软件包安装，编译 | 必须使用 root 权限的操作，需用户保证拥有对应权限，不涉及用户名密码保存。 | +| 补丁反馈 | 提交 issue | 无需用户名密码，需要 token，token 可设置环境变量和写入配置文件，由用户自己保存。 | + +### 3.8、扩展性设计 + +#### 3.8.1、查找平台扩展 + +工具采用 yaml 配置查找平台及对应的查找规则，支持动态扩展，开发者只要按照定义的 yaml 配置文件（cve-tracking.yaml）规则和说明配置对应平台信息即可，当 yaml 配置项不满足时可调整 yaml 解析类进行扩展。 + +```yaml +# 匹配cve补丁信息的正则表达式 +regex: + # 匹配内容的类型的标签，当前只支持 "commit"/"pr"/"issue" + - label: commit + # 匹配特定类型内容的正则表达式，可以设置多个匹配规则 + regular: + - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40} + - http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40} + - label: pr + regular: + - http[s]?://(?:[-\w.\/;?])+(?:pull[s]?|merge_requests)/[1-9][0-9]* + - label: issue + regular: + - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+ + +# 查找cve漏洞修复的平台，例如 Debian、Bugzilla、Nvd等 +platform: + # The name of the platform, similar to a label + # 平台的名称(必配项) + - name: Cnnvd + # Request address to find CVE information,{cve_num} is a placeholder for string substitution in Python + # 查找cve详情信息的请求地址，“{cve_num}”是python中字符串替换的占位符(必配项) + url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num} + # 发送请求的方式，默认为get请求，还可以指定为post请求，当请求方式为get时，此处可不配置 + method: get + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式，当请求方式为get时，此处可不配置 + body: + # 解析响应体的方式，当前只支持“text”或“json”(必配项) + format: text + # 当请求无法直接获取cve信息时（根据页面中特定链接多次跳转），可以指定页面跳转的方式来获取进一步的信息 + redirect: + # 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写 + - prefix: http://cnnvd.org.cn + # 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项 + regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+ + # 送请求的方法，默认为get请求，还可以指定为post请求，当请求为get时，此处可不配置 + method: get + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式 + body: + + - name: Debian + url: https://security-tracker.debian.org/tracker/{cve_num} + format: text + + - name: Ubuntn + url: https://ubuntu.com/security/{cve_num} + format: text + + - name: Bugzilla + url: https://bugzilla.redhat.com/rest/bug/{cve_num}/comment + format: json + + - name: Nvd + url: https://nvd.nist.gov/vuln/detail/{cve_num} + format: text + + - name: Suse + url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num} + format: text + +# Private token for API access +# api访问时的私人令牌，当前只支持"gitee"/"github"/"gitlab" +authentication: + # api名称，此值不可更改 + - name: gitee + # 访问api的私人令牌，可以手动获取后更改 + token: 9c2290c567bd16064219cbbc0f61c8dc + - name: github + token: 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 + - name: gitlab + token: glpat-24DGxkuTA8nTR1YQsXvZ + +# 补丁包下载配置项 +download: + # 补丁包下载路径的调整，分为（REPLACE、APPEND）两种方式，分别对应了补丁路径的替换和拼接 + - action: REPLACE + # 匹配url的正则表达式，符合特定规则的url进行特定值的替换（必填项） + regex: http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40} + # 替换的规则，第一个值为需要替换的内容，第二个值为替换的目标内容，如果不传递默认为（commit,patch）进行替换操作（可选配置项） + target: + - commit + - patch + - action: REPLACE + regex: http[s]?://(?:[-\w.\/;?])+/commit[s]?/\?id=[0-9a-z]{8,40} + target: + - commit + - patch + - action: APPEND + # 匹配url的正则表达式，符合特定规则的url进行特定值的拼接（必填项） + regex: http[s]?://(?:[-\w.\/;?])+/commit[s]?/[0-9a-z]{8,40} + # 拼接的规则，在url地址的末尾需要拼接的内容，如果不传递默认拼接“.patch”(可选配置项) + target: + - .patch +``` + +#### 3.8.2、功能扩展 + +工具采用工厂模式设计模式，当命令行和功能模块扩展时，只需增量添加对应的类即可。 + +![](images\扩展设计.png) + +### 3.9、特性清单 + +**AR：补丁查找** + +| SR | 描述 | 工作量 | +| -----------: | ------------------------------------------------------------ | ------ | +| cve 信息获取 | 从四个常用 cve 参考网址中获取 cve 相关信息。 | 1K | +| 补丁获取 | 整合获取的 cve 信息，提取 commits、pr、issue，分析补丁内容。 | 1k | + +**AR：issue 评论** + +| SR | 描述 | 工作量 | +| -----------: | ----------------------------------------------------------------------------- | ------ | +| 评论内容拼接 | 将获取到的补丁信息，封装拼接成最终需要评论到（cve）issue 中的内容。 | 0.2k | +| 补丁信息评论 | 封装 gitee 对 issue 的评论接口，将拼接好的补丁信息添加到（cve）issue 评论中。 | 0.3k | + +**AR：补丁下载** + +| SR | 描述 | 工作量 | +| -------: | ------------------------------------ | ------ | +| 补丁下载 | 根据获取到的补丁链接，进行下载保存。 | 0.3k | + +**AR：补丁验证** + +| SR | 描述 | 工作量 | +| -------: | --------------------------------------------------------------------------------------------- | ------ | +| 补丁验证 | 根据下载的补丁文件，以及软件包的 gitee 地址，下载源代码，修改 spec 文件，进行软件包打包验证。 | 0.5k | + +### 3.10、接口清单 + +#### 3.10.1、查找补丁信息 find_patches_info() + +> 说明：在各个 cve 参考网址爬取 cve 的补丁信息，先初步记录参考网址上列出的网址。 + +入参： + +| 名称 | 类型 | 说明 | +| --------------- | ---- | -------------------------------------------- | +| cve_num | str | cve 编号 | +| rpm_name | str | rpm 包名称 | +| \*cve_plantform | str | cve 查找参考网址，可使用属性值表示，无需传入 | + +返回值： + +| 名称 | 类型 | 说明 | +| ------------ | ---- | ------------ | +| \*patch_info | dict | 补丁相关信息 | + +patch_info: + +```json +{ + "plantform": "nvd_url" + "commits": ["commit1","commit2"], + "pr": ["pr1","pr2"], + "issue": ["issue1","issue2"] +} +``` + +#### 3.10.2、查找补丁链接 find_patches_detail() + +> 说明：对 find_patches_info 查询的信息做进一步处理，获取补丁链接。 + +入参： + +| 名称 | 类型 | 说明 | +| ----------- | ---------- | ----------------------------------------- | +| patch_infos | list(dict) | find_patches_info()接口查出的补丁信息列表 | + +patch_infos： + +```json +[ +{ + "plantform": "nvd_url" + "commits": ["commit11","commit12"], + "pr": ["pr11","pr12"], + "issue": ["issue11","issue12"] +}, +{ + "plantform": "buzila_url" + "commits": ["commit21","commit22"], + "pr": ["pr21","pr22"], + "issue": ["issue21","issue22"] +} +] +``` + +返回值： + +| 名称 | 类型 | 说明 | +| ------------- | ---------- | -------------------- | +| patch_details | list(dict) | 补丁链接详细信息列表 | + +patch_details： + +```json +[ + { + "plantform": "nvd_url", + "details": { + "issue1": { + "url": "issue1_url", + "status": "close", + "prs": [ + { + "url": "pr1_url", + "status": "open", + "commits": ["commit1", "commit2"] + } + ] + } + } + }, + { + "plantform": "buzila_url", + "details": { + "issue1": { + "url": "", + "status": "", + "prs": [ + { + "url": "", + "status": "", + "commits": ["commit1", "commit2"] + } + ] + } + } + } +] +``` + +#### 3.10.3、评论 issue add_comment() + +> 说明：根据 patch_details 获取的补丁详细信息，拼接 issue 评论内容，添加 issue 评论。 + +入参： + +| 名称 | 类型 | 说明 | +| ------------- | ---------- | ------------------------------------------------------- | +| issue_num | str | issue 编号 | +| patch_details | list(dict) | 补丁链接详细信息列表，结构见 find_patches_detail 返回值 | + +返回值： + +| 名称 | 类型 | 说明 | +| ---- | ---- | ------------------------------------------ | +| code | str | 调用 gitee api 添加 issue 评论返回的状态码 | +| text | str | 调用 gitee api 添加 issue 评论返回的信息 | + +#### 3.10.4、下载保存 save_patch() + +> 说明：根据 find_patches_detail 获取到的补丁信息，进行补丁文件下载。 + +入参： + +| 名称 | 类型 | 说明 | +| ------------- | ---------- | ------------------------------------------------------- | +| patch_details | list(dict) | 补丁链接详细信息列表，结构见 find_patches_detail 返回值 | +| cve_num | str | cve 编号 | +| \*save_path | str | 补丁文件保存路径，可不指定，有默认值。 | + +返回值： + +> 无返回值，下载信息记录日志 + +#### 3.10.5、打包验证 packing_source() + +> 说明：根据 save_patch 保存的补丁文件，以及软件包信息，从 gitee 拉取源代码，修改 spec，进行打包测试。 + +入参： + +| 名称 | 类型 | 说明 | +| ------------- | ---- | ----------------------------------------------- | +| rpm_name | str | 补丁对应的软件包名称，用于从 gitee 中拉取源代码 | +| \*source_path | str | 源码下载路径，可不指定，有默认值。 | +| \*save_path | str | 补丁文件保存路径，可不指定，有默认值。 | + +返回值： + +> 无返回值，编译过程记录文件，编译结果记录日志。 + +#### 3.10.6、补丁信息反馈 create_feedback_issue() + +> 说明：反馈 cve 参考网址和补丁 url 信息，到指定仓库下提交 issue + +入参： + +| 名称 | 类型 | 说明 | +| -------------- | ---- | ----------------------------------------- | +| cve_num | str | cve 编号 | +| cve_platform | str | cve 参考网址 | +| patch_url | str | 补丁 url 链接 | +| issue_platform | str | 提交 issue 的平台 | +| \*owner | str | 提交 issue 所在的 owner，在配置文件中配置 | +| \*repo | str | 提交 issue 所在的 repo, 在配置文件中配置 | diff --git a/cve-agency-manager/cve_tracking/doc/images/image-20211112101004293.png b/cve-agency-manager/cve_tracking/doc/images/image-20211112101004293.png new file mode 100644 index 0000000000000000000000000000000000000000..2bf18cf98cb3fed4514d040f763f576f57087b36 Binary files /dev/null and b/cve-agency-manager/cve_tracking/doc/images/image-20211112101004293.png differ diff --git a/cve-agency-manager/cve_tracking/doc/images/image-20211112171943095.png b/cve-agency-manager/cve_tracking/doc/images/image-20211112171943095.png new file mode 100644 index 0000000000000000000000000000000000000000..fb773c0183a5126b25372cbf10b3d23dbb78e0ee Binary files /dev/null and b/cve-agency-manager/cve_tracking/doc/images/image-20211112171943095.png differ diff --git a/cve-agency-manager/cve_tracking/doc/images/image-20211112172028891.png b/cve-agency-manager/cve_tracking/doc/images/image-20211112172028891.png new file mode 100644 index 0000000000000000000000000000000000000000..da4468ca8eae95be0fedf25b738422669c2232f2 Binary files /dev/null and b/cve-agency-manager/cve_tracking/doc/images/image-20211112172028891.png differ diff --git a/cve-agency-manager/cve_tracking/doc/images/image-20211112172057467.png b/cve-agency-manager/cve_tracking/doc/images/image-20211112172057467.png new file mode 100644 index 0000000000000000000000000000000000000000..ed4888ec7ae8e90871b40925430a092acd18d62e Binary files /dev/null and b/cve-agency-manager/cve_tracking/doc/images/image-20211112172057467.png differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\346\250\241\345\235\227.png" "b/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\346\250\241\345\235\227.png" new file mode 100644 index 0000000000000000000000000000000000000000..1987193dfd20fc69f544a6a98d3102cdf30e9409 Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\346\250\241\345\235\227.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" "b/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" new file mode 100644 index 0000000000000000000000000000000000000000..ed8b9633710875acbf53e360c137567f4ea2d830 Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" new file mode 100644 index 0000000000000000000000000000000000000000..a1da01a4592a933bece141ed77d370be3b86eaa4 Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\346\200\247\350\203\275\347\255\226\347\225\245.png" "b/cve-agency-manager/cve_tracking/doc/images/\346\200\247\350\203\275\347\255\226\347\225\245.png" new file mode 100644 index 0000000000000000000000000000000000000000..a5eccf8b37e0b632f426670a67281b75bb64ea4d Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\346\200\247\350\203\275\347\255\226\347\225\245.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" "b/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" new file mode 100644 index 0000000000000000000000000000000000000000..57922ae14b4161e6be7d2cb2b8b5a833f4f73b4d Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\346\227\266\345\272\217\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\346\227\266\345\272\217\345\233\276.png" new file mode 100644 index 0000000000000000000000000000000000000000..1afce77a7da7c8a5f30334d26453d261910c4222 Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\346\227\266\345\272\217\345\233\276.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" new file mode 100644 index 0000000000000000000000000000000000000000..53cacf2c001132d41489e19b355def525c6f26fc Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" new file mode 100644 index 0000000000000000000000000000000000000000..6edec76befbac3f31b17c8931fbd5b0f211f4998 Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" differ diff --git a/cve-agency-manager/cve_tracking/exception.py b/cve-agency-manager/cve_tracking/exception.py new file mode 100644 index 0000000000000000000000000000000000000000..684712e2039304337ea2e8e32c664ede3314ad34 --- /dev/null +++ b/cve-agency-manager/cve_tracking/exception.py @@ -0,0 +1,68 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +class CveException(BaseException): + """ + The parent exception class of the cve tracking tool + """ + + def __init__(self, msg): + super(CveException, self).__init__() + self.message = msg + + def __repr__(self): + return self.message + + +class ConfigNotFoundError(CveException): + """ + Configuration file not found exception + """ + + def __init__(self, msg): + super(ConfigNotFoundError, self).__init__(f'Configuration file not found: {msg}') + + +class InputError(CveException): + """ + Input params error + """ + + def __init__(self, msg): + super(InputError, self).__init__(f'Input parameter error: {msg}') + + +class RequestError(CveException): + """ + Error calling URL + """ + + def __init__(self, msg): + super(RequestError, self).__init__(f'Error calling URL: {msg}') + + +class PathNotExistError(CveException): + """ + Path not exist error + """ + + def __init__(self, msg): + super(PathNotExistError, self).__init__(f'Path not exist: {msg}') + + +class PathEmptyError(CveException): + """ + Path is empty error + """ + + def __init__(self, msg): + super(PathEmptyError, self).__init__(f'Path is empty: {msg}') diff --git a/cve-agency-manager/cve_tracking/logger.py b/cve-agency-manager/cve_tracking/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..87e21ebb7d5126992f798e68be71f66439184644 --- /dev/null +++ b/cve-agency-manager/cve_tracking/logger.py @@ -0,0 +1,74 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import logging +import os.path + +from concurrent_log_handler import ConcurrentRotatingFileHandler + +from conf import CONFIG + + +class Logger: + """ + Log type, including output to console, output to file, log file dump. + This class uses the singleton pattern + """ + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls, *args, **kwargs) + return cls._instance + + def __init__(self): + self._logger = logging.getLogger(self.__class__.__name__) + self._log_format = logging.Formatter( + '%(asctime)s-[%(name)s]-[file:%(filename)s:%(lineno)d]-[function:%(funcName)s]-[%(levelname)s]: %(message)s' + , datefmt='%a, %d %b %Y %H:%M:%S') + + def init_logger(self): + """ + Set a custom format for log + :return: self._logger + """ + self._logger.setLevel(CONFIG.LOGGER_LEVEL) + self._add_handler() + return self._logger + + def _add_handler(self): + """ + Add output stream for log, including console output and file output + :return: None + """ + # add console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(CONFIG.LOGGER_LEVEL) + console_handler.setFormatter(self._log_format) + self._logger.addHandler(console_handler) + # create log path + if not os.path.exists(CONFIG.LOGGER_PATH): + os.mkdir(CONFIG.LOGGER_PATH, mode=0x644) + log_file = os.path.join(CONFIG.LOGGER_PATH, CONFIG.LOGGER_FILE_NAME) + # add file handler + file_handler = ConcurrentRotatingFileHandler(filename=log_file, + mode='a', + maxBytes=CONFIG.LOGGER_BUFFER, + backupCount=CONFIG.LOGGER_FILE_COUNT, + encoding="utf-8", + use_gzip=True) + file_handler.setLevel(CONFIG.LOGGER_LEVEL) + file_handler.setFormatter(self._log_format) + self._logger.addHandler(file_handler) + + +logger = Logger().init_logger() diff --git a/cve-agency-manager/cve_tracking/main.py b/cve-agency-manager/cve_tracking/main.py new file mode 100644 index 0000000000000000000000000000000000000000..88cf3239db7b6e967da680881a13c0a6218835cc --- /dev/null +++ b/cve-agency-manager/cve_tracking/main.py @@ -0,0 +1,33 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +Tool execution total entrance +""" +import asyncio + +from cli.base import CveTrackingCommand +from conf import CONFIG +from exception import CveException +from logger import logger + +try: + # Register all subcommands + for sub_class in CveTrackingCommand.__subclasses__(): + CveTrackingCommand.register_command(sub_class()) + + # Parallel execution of corresponding functions + loop = asyncio.get_event_loop() + loop.run_until_complete(CveTrackingCommand.run()) +except CveException as e: + logger.error( + f'Execution failed, message is : {e.message}, detail in {CONFIG.LOGGER_PATH}/{CONFIG.LOGGER_FILE_NAME}') diff --git a/cve-agency-manager/cve_tracking/request.py b/cve-agency-manager/cve_tracking/request.py new file mode 100644 index 0000000000000000000000000000000000000000..b2859f473cc90ed60456d3495f95ef8da3b6bbe1 --- /dev/null +++ b/cve-agency-manager/cve_tracking/request.py @@ -0,0 +1,155 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import json +from asyncio import TimeoutError +from http.client import RemoteDisconnected +from urllib.error import HTTPError, URLError + +import aiohttp +import wget +from fake_useragent import UserAgent +from retrying import retry + +from constant import Constant +from exception import RequestError + + +class Response: + def __init__(self, error=None, response: aiohttp.ClientResponse = None, text=None) -> None: + self.error = error + self.response = response + self.text = text + + @property + def success(self): + return self.response and (self.response.status == 200 or self.response.status == 201) + + @property + def status_code(self): + return self.response and self.response.status + + @property + def json(self): + if not self.response: + return None + try: + return json.loads(self.text) + except json.JSONDecodeError: + return None + + +class Http: + """ + async http request + """ + user_agent = UserAgent() + session = None + + def __init__(self, max_attempt_number=Constant.MAX_RETRY, stop_max_delay=Constant.MAX_DELAY) -> None: + if self.session is None: + self.session = aiohttp.ClientSession() + self._max_attempt_number = max_attempt_number + self._stop_max_delay = stop_max_delay + self._response = Response() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.session.close() + + @property + def headers(self): + headers = { + "User-Agent": self.user_agent.random + } + return headers + + def _set_headers(self, request_header): + if "headers" not in request_header: + request_header["headers"] = self.headers + request_header["headers"].setdefault( + "User-Agent", self.user_agent.random) + request_header["headers"].setdefault( + "Content-Type", "application/json") + + async def _get(self, url, params=None, **kwargs): + self._set_headers(request_header=kwargs) + + @retry(stop_max_attempt_number=self._max_attempt_number, stop_max_delay=self._stop_max_delay) + async def _http_request(): + return await self.session.get(url, params=params, timeout=Constant.CALL_MAX_DELAY, **kwargs) + + await self._request(_http_request) + + @classmethod + async def get(cls, url, **kwargs) -> "Response": + async with cls() as self: + await self._get(url, **kwargs) + return self._response + + async def _request(self, http_request): + try: + self._response.response = await http_request() + self._response.text = await self._response.response.text() + except aiohttp.ClientError as error: + self._response.error = error + except TimeoutError as e: + raise RequestError(f'Call url timeout: {e}') + + async def _post(self, url, data=None, **kwargs): + # self._set_headers(request_header=kwargs) + + @retry(stop_max_attempt_number=self._max_attempt_number, stop_max_delay=self._stop_max_delay) + async def _http_request(): + return await self.session.post(url, data=data, timeout=Constant.CALL_MAX_DELAY, **kwargs) + + await self._request(_http_request) + + @classmethod + async def post(cls, url, data=None, **kwargs) -> "Response": + async with cls() as self: + await self._post(url, data, **kwargs) + return self._response + + def _dw(self, url, out_fname): + @retry(stop_max_attempt_number=self._max_attempt_number, stop_max_delay=self._stop_max_delay) + def getfile(): + file_name = wget.download(url, out_fname) + if not file_name: + raise + return file_name + + return getfile() + + @staticmethod + def download(url, out_fname, **kwargs): + max_attempt_number = kwargs.get("max_attempt_number", Constant.MAX_RETRY) + stop_max_delay = kwargs.get("stop_max_delay", Constant.MAX_DOWNLOAD_DELAY) + + @retry(stop_max_attempt_number=max_attempt_number, stop_max_delay=stop_max_delay) + def _dw(): + file_name = wget.download(url, out_fname) + if not file_name: + raise RequestError(f'Call url:"{url}" failed') + return file_name + + try: + return _dw() + except (RequestError, RemoteDisconnected, HTTPError, URLError): + return None + + +http = Http + +__all__ = ("http",) diff --git a/cve-agency-manager/cve_tracking/requirements.txt b/cve-agency-manager/cve_tracking/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..99850e426a2b7813c13c6c4c2aca28681b20da15 --- /dev/null +++ b/cve-agency-manager/cve_tracking/requirements.txt @@ -0,0 +1,9 @@ +asyncio +bs4 +aiohttp +wget +fake_useragent +retrying +concurrent_log_handler +lxml +pyyaml \ No newline at end of file diff --git a/cve-agency-manager/cve_tracking/util/__init__.py b/cve-agency-manager/cve_tracking/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5a7ebcdc275716a52600da9e60838866db1f34a9 --- /dev/null +++ b/cve-agency-manager/cve_tracking/util/__init__.py @@ -0,0 +1,67 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from exception import RequestError +from request import http +from logger import logger + + +class Api: + + def __init__(self, repo=None, owner=None, token=None) -> None: + self._repo = repo + self._owner = owner + self.token = token + + def _set_token(self): + parameters = {"access_token": self.token} if self.token else {} + return parameters + + @staticmethod + async def _post(url, values, **kwargs): + """ + POST into gitee API + """ + try: + response = await http.post(url=url, data=values, **kwargs) + except RequestError as e: + logger.error(f'Call url {url} failed, message: {e.message}') + return None + + if not response.success: + logger.warning("reuqest url: %s status code: %s error info: %s" + % (url, response.status_code, response.error)) + return None + + return response.json + + def set_attr(self, owner, repo): + self._repo = repo + self._owner = owner + + @staticmethod + async def _get(url, params=None, text=False, **kwargs): + """ + GET from gitee api + """ + try: + response = await http.get(url=url, params=params, **kwargs) + except RequestError as e: + logger.error(f'Call url {url} failed, message: {e.message}') + return None + + if not response.success: + logger.warning("reuqest url: %s status code: %s error info: %s" + % (url, response.status_code, response.error)) + return None + + return response.text if text else response.json diff --git a/cve-agency-manager/cve_tracking/util/gitee_api.py b/cve-agency-manager/cve_tracking/util/gitee_api.py new file mode 100644 index 0000000000000000000000000000000000000000..24aa3151256d5411f2799a31fbe2f666c59486ca --- /dev/null +++ b/cve-agency-manager/cve_tracking/util/gitee_api.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +This is a helper script for working with gitee.com +""" +import os + +from conf import settings +from exception import ConfigNotFoundError +from logger import logger +from . import Api + + +class Gitee(Api): + """ + Gitee is a helper class to abstract gitee.com api + """ + host = "https://gitee.com/api/v5/repos" + + def __init__(self): + super(Gitee, self).__init__() + self.token = os.getenv("TRACK_GITEE_TOKEN", settings.token(name="gitee")) + if not self.token: + raise ConfigNotFoundError( + "Please set the 'TRACK_GITEE_TOKEN' environment variables of gitee." + ) + + async def create_issue(self, title, body): + """ + Create issue in gitee + """ + issues_url = f"{self.host}/{self._owner}/issues" + parameters = self._set_token() + parameters["repo"] = self._repo + parameters["title"] = title + parameters["body"] = body + return await self._post(issues_url, parameters) + + async def create_issue_comment(self, number, body): + """ + create issue comment + """ + url = f"{self.host}/{self._owner}/{self._repo}/issues/{number}/comments" + values = self._set_token() + values["body"] = body + logger.info(f"Add comment to issue url: {url}") + return await self._post(url, values) + + async def create_pr_comment(self, number, body): + """ + Post comment to the given specific PR + """ + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{number}/comments" + values = self._set_token() + values["body"] = body + return self._post(url, values) diff --git a/cve-agency-manager/cve_tracking/util/github_api.py b/cve-agency-manager/cve_tracking/util/github_api.py new file mode 100644 index 0000000000000000000000000000000000000000..b607a3c3b85c745403894ab66b46dbce8d3027ae --- /dev/null +++ b/cve-agency-manager/cve_tracking/util/github_api.py @@ -0,0 +1,133 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import os + +from lxml import etree + +from conf import settings +from . import Api + + +class Github(Api): + host = "https://api.github.com/repos" + + def __init__(self) -> None: + super(Github, self).__init__() + self.token = os.getenv("GITHUB_TOKEN", settings.token(name="github")) + + @property + def _headers(self): + headers = { + "Accept": "application/vnd.github.v3+json", + } + if self.token: + headers["Authorization"] = f"token {self.token}" + return headers + + async def get_pull_commits(self, pull_number, page=100, curpage=1): + """ + Get the commit contained in pr + :param pull_number: pull number + :param page: page number + :param curpage: current page number + :return: response + """ + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{pull_number}/commits" + return await self._get( + url, params=dict(per_page=page, page=curpage), headers=self._headers + ) + + async def get_pull_request(self, pull_number): + """ + Get a pull request + """ + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{pull_number}" + return await self._get(url, headers=self._headers) + + async def get_issue(self, issue_number): + """ + Get issue information + :param issue_number: issue num + :return: response + """ + url = f"{self.host}/{self._owner}/{self._repo}/issues/{issue_number}" + return await self._get(url, headers=self._headers) + + async def get_issue_comments(self, issue_number): + """ + Get issue comments + :param issue_number: issue num + :return: response + """ + url = f"{self.host}/{self._owner}/{self._repo}/issues/{issue_number}/comments" + + return await self._get(url, headers=self._headers) + + async def create_issue(self, title, body): + url = f"{self.host}/{self._owner}/{self._repo}/issues" + params = {"title": title, "body": body} + return await self._post( + url=url, values=None, json=params, headers=self._headers + ) + + async def check_pull_merged(self, pull_number): + """ + Check pr status + :param pull_number: pull number + :return: response + """ + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{pull_number}/merge" + response = await self._get(url, headers=self._headers) + if response.status_code == 204: + return True + return False + + @staticmethod + def _pull(pulls): + """ + Splicing pull + :param pulls: pull url + :return: full url + """ + return ["https://github.com" + pr for pr in pulls] + + async def issue_relevance_pull(self, issue_number=None, issue_url=None): + """ + Crawl issue URL information, get associated pr + :param issue_number: issue num + :param issue_url: issue url + :return: pull list + """ + if issue_url: + response = await self._get(url=issue_url, text=True) + else: + response = await self._get( + url=f"https://github.com/{self._owner}/{self._repo}/issues/{issue_number}", + text=True, + ) + if not response: + return [] + + html = etree.HTML(response) + link = html.xpath( + "//div[contains(@class,'discussion-sidebar-item') and contains(@class,'js-discussion-sidebar-item') " + "and last() ]/form/div[contains(@class,'css-truncate')]" + ) + if link: + return self._pull(pulls=link[-1].xpath("./a/@href")) + + # Search for operation logs + pulls = html.xpath( + "//div[@class='TimelineItem']/div[@class='TimelineItem-body']//div[contains(@class,'flex-auto')]/a/@href" + ) + return self._pull(pulls=pulls) diff --git a/cve-agency-manager/cve_tracking/util/gitlab_api.py b/cve-agency-manager/cve_tracking/util/gitlab_api.py new file mode 100644 index 0000000000000000000000000000000000000000..a127e8a63741bc46079e6a965818bf00e4b9ad03 --- /dev/null +++ b/cve-agency-manager/cve_tracking/util/gitlab_api.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import os + +from exception import ConfigNotFoundError +from conf import CONFIG, settings +from . import Api + + +class Gitlab(Api): + host = "https://gitlab.com/api/v4/projects" + + def __init__(self) -> None: + if not settings.token(name="gitlab") and not os.getenv("GITLAB-TOKEN", None): + raise ConfigNotFoundError( + "Please set the 'GITLAB-TOKEN' environment variable for gitlab." + ) + super(Gitlab, self).__init__() + + @property + def _headers(self): + return {"PRIVATE-TOKEN": os.getenv("GITLAB-TOKEN", settings.token('gitlab'))} + + async def get_commit_comments(self, commit_sha): + """ + Get the comments of a commit in a project. + + :param commit_sha: The commit hash or name of a repository branch or tag + """ + url = f"{self.host}/{self._owner}%2F{self._repo}/repository/commits/{commit_sha}/comments" + return await self._get(url, headers=self._headers) + + async def get_single_mr(self, merge_request_iid): + """ + Shows information about a single merge request. + + :param merge_request_iid: The internal ID of the merge request. + """ + url = f"{self.host}/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}" + return await self._get(url, headers=self._headers) + + async def get_mr_comments(self, merge_request_iid): + """ + Gets a list of all notes for a single merge request. + + :param merge_request_iid: The internal ID of the merge request. + """ + + url = f"{self.host}/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}/notes" + return await self._get(url, headers=self._headers) + + async def get_mr_context_commits(self, merge_request_iid): + """ + Get a list of merge request context commits. + + :param merge_request_iid: The internal ID of the merge request. + """ + + url = f"{self.host}/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}/commits" + + return await self._get(url, headers=self._headers) + + async def issue_relevance_pull(self, issue_id): + """ + Get all merge requests that close a particular issue when merged. + + :param issue_id: The internal ID of a project issue + """ + url = f"{self.host}/{self._owner}%2F{self._repo}/issues/{issue_id}/closed_by" + return await self._get(url, headers=self._headers) + + async def create_issue(self, title, body): + """ + Creates a new project issue. + :param title: issue title + :param body: issue body + :return: + """ + url = f"{self.host}/{self._owner}%2F{self._repo}/issues" + params = {'title': title, + 'description': body} + return await self._post(url, headers=self._headers, values=params) diff --git a/cve-agency-manager/go.mod b/cve-agency-manager/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..4d63604fc09005b77a0bc28728bb7ccc9650d20f --- /dev/null +++ b/cve-agency-manager/go.mod @@ -0,0 +1,8 @@ +module cve-agency-manager + +go 1.14 + +require ( + github.com/astaxie/beego v1.12.3 + github.com/smartystreets/goconvey v1.7.2 +) diff --git a/cve-agency-manager/go.sum b/cve-agency-manager/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..6a80d03cf78b8e26dc6928f1f38f09336faee65f --- /dev/null +++ b/cve-agency-manager/go.sum @@ -0,0 +1,188 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= +github.com/astaxie/beego v1.12.3 h1:SAQkdD2ePye+v8Gn1r4X6IKZM1wd28EyUOVQ3PDSOOQ= +github.com/astaxie/beego v1.12.3/go.mod h1:p3qIm0Ryx7zeBHLljmd7omloyca1s4yu1a8kM1FkpIA= +github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ= +github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= +github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= +github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= +github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= +github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= +github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U= +github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo= +github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= +github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/siddontang/goredis v0.0.0-20150324035039-760763f78400/go.mod h1:DDcKzU3qCuvj/tPnimWSsZZzvk9qvkvrIL5naVBPh5s= +github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= +github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/cve-agency-manager/main.go b/cve-agency-manager/main.go new file mode 100644 index 0000000000000000000000000000000000000000..de0651388fe165f51dc904e91fd3d74ae5a44345 --- /dev/null +++ b/cve-agency-manager/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "cve-agency-manager/common" + _ "cve-agency-manager/routers" + "github.com/astaxie/beego" +) + +func init() { + // Initialization log + common.LogInit() +} + +func main() { + if beego.BConfig.RunMode == "dev" { + beego.BConfig.WebConfig.DirectoryIndex = true + beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" + } + beego.Run() +} diff --git a/cve-agency-manager/models/object.go b/cve-agency-manager/models/object.go new file mode 100644 index 0000000000000000000000000000000000000000..3ffb60850f104eabf7ca3d29fda97cf49c258d16 --- /dev/null +++ b/cve-agency-manager/models/object.go @@ -0,0 +1,53 @@ +package models + +import ( + "errors" + "strconv" + "time" +) + +var ( + Objects map[string]*Object +) + +type Object struct { + ObjectId string + Score int64 + PlayerName string +} + +func init() { + Objects = make(map[string]*Object) + Objects["hjkhsbnmn123"] = &Object{"hjkhsbnmn123", 100, "astaxie"} + Objects["mjjkxsxsaa23"] = &Object{"mjjkxsxsaa23", 101, "someone"} +} + +func AddOne(object Object) (ObjectId string) { + object.ObjectId = "astaxie" + strconv.FormatInt(time.Now().UnixNano(), 10) + Objects[object.ObjectId] = &object + return object.ObjectId +} + +func GetOne(ObjectId string) (object *Object, err error) { + if v, ok := Objects[ObjectId]; ok { + return v, nil + } + return nil, errors.New("ObjectId Not Exist") +} + +func GetAll() map[string]*Object { + return Objects +} + +func Update(ObjectId string, Score int64) (err error) { + if v, ok := Objects[ObjectId]; ok { + v.Score = Score + return nil + } + return errors.New("ObjectId Not Exist") +} + +func Delete(ObjectId string) { + delete(Objects, ObjectId) +} + diff --git a/cve-agency-manager/models/user.go b/cve-agency-manager/models/user.go new file mode 100644 index 0000000000000000000000000000000000000000..d4bebb207c346beb63897d800b69dd6b5d223b3e --- /dev/null +++ b/cve-agency-manager/models/user.go @@ -0,0 +1,86 @@ +package models + +import ( + "errors" + "strconv" + "time" +) + +var ( + UserList map[string]*User +) + +func init() { + UserList = make(map[string]*User) + u := User{"user_11111", "astaxie", "11111", Profile{"male", 20, "Singapore", "astaxie@gmail.com"}} + UserList["user_11111"] = &u +} + +type User struct { + Id string + Username string + Password string + Profile Profile +} + +type Profile struct { + Gender string + Age int + Address string + Email string +} + +func AddUser(u User) string { + u.Id = "user_" + strconv.FormatInt(time.Now().UnixNano(), 10) + UserList[u.Id] = &u + return u.Id +} + +func GetUser(uid string) (u *User, err error) { + if u, ok := UserList[uid]; ok { + return u, nil + } + return nil, errors.New("User not exists") +} + +func GetAllUsers() map[string]*User { + return UserList +} + +func UpdateUser(uid string, uu *User) (a *User, err error) { + if u, ok := UserList[uid]; ok { + if uu.Username != "" { + u.Username = uu.Username + } + if uu.Password != "" { + u.Password = uu.Password + } + if uu.Profile.Age != 0 { + u.Profile.Age = uu.Profile.Age + } + if uu.Profile.Address != "" { + u.Profile.Address = uu.Profile.Address + } + if uu.Profile.Gender != "" { + u.Profile.Gender = uu.Profile.Gender + } + if uu.Profile.Email != "" { + u.Profile.Email = uu.Profile.Email + } + return u, nil + } + return nil, errors.New("User Not Exist") +} + +func Login(username, password string) bool { + for _, u := range UserList { + if u.Username == username && u.Password == password { + return true + } + } + return false +} + +func DeleteUser(uid string) { + delete(UserList, uid) +} diff --git a/cve-agency-manager/routers/commentsRouter_controllers.go b/cve-agency-manager/routers/commentsRouter_controllers.go new file mode 100644 index 0000000000000000000000000000000000000000..318d81ff372ec56cc6f2b60511f583c6bdc1a72f --- /dev/null +++ b/cve-agency-manager/routers/commentsRouter_controllers.go @@ -0,0 +1,145 @@ +package routers + +import ( + "github.com/astaxie/beego" + "github.com/astaxie/beego/context/param" +) + +func init() { + + beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"], + beego.ControllerComments{ + Method: "GetAll", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Get", + Router: "/:objectId", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Put", + Router: "/:objectId", + AllowHTTPMethods: []string{"put"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Delete", + Router: "/:objectId", + AllowHTTPMethods: []string{"delete"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:TriggerHeartbeatLiveController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:TriggerHeartbeatLiveController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:TriggerHeartbeatreadController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:TriggerHeartbeatreadController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:TriggerTrackController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:TriggerTrackController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "GetAll", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "Get", + Router: "/:uid", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "Put", + Router: "/:uid", + AllowHTTPMethods: []string{"put"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "Delete", + Router: "/:uid", + AllowHTTPMethods: []string{"delete"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "Login", + Router: "/login", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"] = append(beego.GlobalControllerRouter["cve-agency-manager/controllers:UserController"], + beego.ControllerComments{ + Method: "Logout", + Router: "/logout", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + +} diff --git a/cve-agency-manager/routers/router.go b/cve-agency-manager/routers/router.go new file mode 100644 index 0000000000000000000000000000000000000000..4f5360f5005ce9b1d92263682be4eda67c604eda --- /dev/null +++ b/cve-agency-manager/routers/router.go @@ -0,0 +1,44 @@ +// @APIVersion 1.0.0 +// @Title beego Test API +// @Description beego has a very cool tools to autogenerate documents for your API +// @Contact astaxie@gmail.com +// @TermsOfServiceUrl http://beego.me/ +// @License Apache 2.0 +// @LicenseUrl http://www.apache.org/licenses/LICENSE-2.0.html +package routers + +import ( + "cve-agency-manager/controllers" + "github.com/astaxie/beego" +) + +func init() { + ns := beego.NewNamespace("/v1", + beego.NSNamespace("/object", + beego.NSInclude( + &controllers.ObjectController{}, + ), + ), + beego.NSNamespace("/user", + beego.NSInclude( + &controllers.UserController{}, + ), + ), + beego.NSNamespace("/cve/track", + beego.NSInclude( + &controllers.TriggerTrackController{}, + ), + ), + beego.NSNamespace("/cve/healthz/readiness", + beego.NSInclude( + &controllers.TriggerHeartbeatreadController{}, + ), + ), + beego.NSNamespace("/cve/healthz/liveness", + beego.NSInclude( + &controllers.TriggerHeartbeatLiveController{}, + ), + ), + ) + beego.AddNamespace(ns) +} diff --git a/cve-agency-manager/tests/default_test.go b/cve-agency-manager/tests/default_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a6df2619f0ef76bc4eb24d9fb90c0e4e8ac26db3 --- /dev/null +++ b/cve-agency-manager/tests/default_test.go @@ -0,0 +1,38 @@ +package test + +import ( + "net/http" + "net/http/httptest" + "testing" + "runtime" + "path/filepath" + _ "cve-agency-service/routers" + + "github.com/astaxie/beego" + . "github.com/smartystreets/goconvey/convey" +) + +func init() { + _, file, _, _ := runtime.Caller(0) + apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".." + string(filepath.Separator)))) + beego.TestBeegoInit(apppath) +} + +// TestGet is a sample to run an endpoint test +func TestGet(t *testing.T) { + r, _ := http.NewRequest("GET", "/v1/object", nil) + w := httptest.NewRecorder() + beego.BeeApp.Handlers.ServeHTTP(w, r) + + beego.Trace("testing", "TestGet", "Code[%d]\n%s", w.Code, w.Body.String()) + + Convey("Subject: Test Station Endpoint\n", t, func() { + Convey("Status Code Should Be 200", func() { + So(w.Code, ShouldEqual, 200) + }) + Convey("The Result Should Not Be Empty", func() { + So(w.Body.Len(), ShouldBeGreaterThan, 0) + }) + }) +} + diff --git a/cve-vulner-manager/.gitignore b/cve-vulner-manager/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..cc9cc23448ee55b7fcac6cdde06ec7d266fdd5cc --- /dev/null +++ b/cve-vulner-manager/.gitignore @@ -0,0 +1,21 @@ +__pycache__ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.pyc +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +.idea/ +.git/ +.gitee/ +logs/ +lastupdate.tmp diff --git a/cve-vulner-manager/Dockerfile b/cve-vulner-manager/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6508e4777c30b8aa4a8f03fb06e8f69bfa35ef6f --- /dev/null +++ b/cve-vulner-manager/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:latest as BUILDER +LABEL maintainer="zhangjianjun" + +# build binary +RUN mkdir -p /go/src/gitee.com/openeuler/cve-manager +COPY . /go/src/gitee.com/openeuler/cve-manager +RUN cd /go/src/gitee.com/openeuler/cve-manager && CGO_ENABLED=1 go build -v -o ./cve-manager main.go + +# copy binary config and utils +FROM openeuler/openeuler:21.03 +RUN mkdir -p /opt/app/conf/ +COPY ./conf/product_app.conf /opt/app/conf/app.conf +# overwrite config yaml +COPY --from=BUILDER /go/src/gitee.com/openeuler/cve-manager/cve-manager /opt/app + +WORKDIR /opt/app/ +ENTRYPOINT ["/opt/app/cve-manager"] \ No newline at end of file diff --git a/cve-vulner-manager/LICENSE b/cve-vulner-manager/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a10985f0870b66a8385e18df5fe94ec35ff5919a --- /dev/null +++ b/cve-vulner-manager/LICENSE @@ -0,0 +1,122 @@ +木兰宽松许可证， 第1版 + +2019年8月 http://license.coscl.org.cn/MulanPSL + +您对“软件”的复制、使用、修改及分发受木兰宽松许可证，第1版（“本许可证”）的如下条款的约束： + +0. 定义 + +“软件”是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。 + +“贡献者”是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。 + +“法人实体”是指提交贡献的机构及其“关联实体”。 + +“关联实体”是指，对“本许可证”下的一方而言，控制、受控制或与其共同受控制的机构，此处的控制是指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。 + +“贡献”是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。 + +1. 授予版权许可 + +每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可以复制、使用、修改、分发其“贡献”，不论修改与否。 + +2. 授予专利许可 + +每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定撤销除外）专利许可，供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软件”结合而将必然会侵犯的专利权利要求，不包括仅因您或他人修改“贡献”或其他结合而将必然会侵犯到的专利权利要求。如您或您的“关联实体”直接或间接地（包括通过代理、专利被许可人或受让人），就“软件”或其中的“贡献”对任何人发起专利侵权诉讼（包括反诉或交叉诉讼）或其他专利维权行动，指控其侵犯专利权，则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权行动之日终止。 + +3. 无商标许可 + +“本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可，但您为满足第4条规定的声明义务而必须使用除外。 + +4. 分发限制 + +您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发，不论修改与否，但您必须向接收者提供“本许可证”的副本，并保留“软件”中的版权、商标、专利及免责声明。 + +5. 免责声明与责任限制 + +“软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下，“贡献者”或版权所有者不对任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任，不论因何种原因导致或者基于何种法律理论,即使其曾被建议有此种损失的可能性。 + +条款结束 + +如何将木兰宽松许可证，第1版，应用到您的软件 + +如果您希望将木兰宽松许可证，第1版，应用到您的新软件，为了方便接收者查阅，建议您完成如下三步： + +1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字； + +2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中； + +3， 请将如下声明文本放入每个源文件的头部注释中。 + +Copyright (c) [2019] [name of copyright holder] +[Software Name] is licensed under the Mulan PSL v1. +You can use this software according to the terms and conditions of the Mulan PSL v1. +You may obtain a copy of Mulan PSL v1 at: + http://license.coscl.org.cn/MulanPSL +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +PURPOSE. +See the Mulan PSL v1 for more details. + + + +Mulan Permissive Software License，Version 1 + +Mulan Permissive Software License，Version 1 (Mulan PSL v1) + +August 2019 http://license.coscl.org.cn/MulanPSL + +Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v1 (this License) with following terms and conditions: + +0. Definition + +Software means the program and related documents which are comprised of those Contribution and licensed under this License. + +Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License. + +Legal Entity means the entity making a Contribution and all its Affiliates. + +Affiliates means entities that control, or are controlled by, or are under common control with a party to this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity. + +Contribution means the copyrightable work licensed by a particular Contributor under this License. + +1. Grant of Copyright License + +Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not. + +2. Grant of Patent License + +Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed, excluding of any patent claims solely be infringed by your or others’ modification or other combinations. If you or your Affiliates directly or indirectly (including through an agent, patent licensee or assignee）, institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken. + +3. No Trademark License + +No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in section 4. + +4. Distribution Restriction + +You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software. + +5. Disclaimer of Warranty and Limitation of Liability + +The Software and Contribution in it are provided without warranties of any kind, either express or implied. In no event shall any Contributor or copyright holder be liable to you for any damages, including, but not limited to any direct, or indirect, special or consequential damages arising from your use or inability to use the Software or the Contribution in it, no matter how it’s caused or based on which legal theory, even if advised of the possibility of such damages. + +End of the Terms and Conditions + +How to apply the Mulan Permissive Software License，Version 1 (Mulan PSL v1) to your software + +To apply the Mulan PSL v1 to your work, for easy identification by recipients, you are suggested to complete following three steps: + +1. Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner; +2. Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package; +3. Attach the statement to the appropriate annotated syntax at the beginning of each source file. + +Copyright (c) [2019] [name of copyright holder] +[Software Name] is licensed under the Mulan PSL v1. +You can use this software according to the terms and conditions of the Mulan PSL v1. +You may obtain a copy of Mulan PSL v1 at: + http://license.coscl.org.cn/MulanPSL +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +PURPOSE. +See the Mulan PSL v1 for more details. + diff --git a/cve-vulner-manager/README.en.md b/cve-vulner-manager/README.en.md new file mode 100644 index 0000000000000000000000000000000000000000..701bdc5edb1123b977eebe9b7c185315d285c1ba --- /dev/null +++ b/cve-vulner-manager/README.en.md @@ -0,0 +1,33 @@ +# cve-manager + +## Introduction + +This repository is used to address the code of openEuler cve manager. + +## Architecture + + + +## Prerequisites + +You'll need to setup a MySQL Database before you are getting started. +This is an example to create Database instance. + +* Setup MySQL instance by the Huawei Cloud Relational Database Service (RDS) +* Login in MySQL with your account and password +* Create database instance by running the following command + ``` + CREATE DATABASE cvevulner; + source ./doc/sql/db_struct.sql; + ``` +The information of database instance will be used in the following Installation. + +## Getting Started + +* [Build the project](doc/md/installx.md) + +## Process processing help + +See the [Process processing help](doc/md/manual.md) file for details. +> For Chinese version, please refer to [流程处理帮助](doc/md/manual.md). + diff --git a/cve-vulner-manager/README.md b/cve-vulner-manager/README.md new file mode 100644 index 0000000000000000000000000000000000000000..701bdc5edb1123b977eebe9b7c185315d285c1ba --- /dev/null +++ b/cve-vulner-manager/README.md @@ -0,0 +1,33 @@ +# cve-manager + +## Introduction + +This repository is used to address the code of openEuler cve manager. + +## Architecture + + + +## Prerequisites + +You'll need to setup a MySQL Database before you are getting started. +This is an example to create Database instance. + +* Setup MySQL instance by the Huawei Cloud Relational Database Service (RDS) +* Login in MySQL with your account and password +* Create database instance by running the following command + ``` + CREATE DATABASE cvevulner; + source ./doc/sql/db_struct.sql; + ``` +The information of database instance will be used in the following Installation. + +## Getting Started + +* [Build the project](doc/md/installx.md) + +## Process processing help + +See the [Process processing help](doc/md/manual.md) file for details. +> For Chinese version, please refer to [流程处理帮助](doc/md/manual.md). + diff --git a/cve-vulner-manager/common/aes.go b/cve-vulner-manager/common/aes.go new file mode 100644 index 0000000000000000000000000000000000000000..bd7e5e465329518564a04cedcbb80134a16851f7 --- /dev/null +++ b/cve-vulner-manager/common/aes.go @@ -0,0 +1,200 @@ +package common + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "encoding/base64" + "errors" + "flag" + "fmt" + "github.com/astaxie/beego/logs" + jwt "github.com/dgrijalva/jwt-go" + "math/rand" + "time" +) + +//PKCS7Padding PKCS7 padding mode +func PKCS7Padding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(ciphertext, padtext...) +} + +// PKCS7UnPadding reverse operation of padding, delete padding string +func PKCS7UnPadding(origData []byte) ([]byte, error) { + length := len(origData) + if length == 0 { + return nil, errors.New("加密字符串错误！") + } + // Get the length of the padding string + unPadding := int(origData[length-1]) + // Intercept the slice, delete the padding bytes, and return the plaintext + return origData[:(length - unPadding)], nil + +} + +//AesEcrypt Implement encryption +func AesEcrypt(origData []byte, key []byte) ([]byte, error) { + // Create an instance of an encryption algorithm + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + blockSize := block.BlockSize() + origData = PKCS7Padding(origData, blockSize) + blocMode := cipher.NewCBCEncrypter(block, key[:blockSize]) + crypted := make([]byte, len(origData)) + blocMode.CryptBlocks(crypted, origData) + return crypted, nil +} + +//AesDeCrypt Realize decryption +func AesDeCrypt(cypted []byte, key []byte) ([]byte, error) { + //Create an instance of an encryption algorithm + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + blockSize := block.BlockSize() + blockMode := cipher.NewCBCDecrypter(block, key[:blockSize]) + origData := make([]byte, len(cypted)) + blockMode.CryptBlocks(origData, cypted) + origData, err = PKCS7UnPadding(origData) + if err != nil { + return nil, err + } + return origData, err +} + +//EnPwdCode Encrypted base64 +func EnPwdCode(pwd []byte, key []byte) (string, error) { + result, err := AesEcrypt(pwd, key) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(result), err +} + +//DePwdCode decryption +func DePwdCode(pwd string, key []byte) ([]byte, error) { + //Decrypt base64 string + pwdByte, err := base64.StdEncoding.DecodeString(pwd) + if err != nil { + return nil, err + } + return AesDeCrypt(pwdByte, key) + +} + +var ( + length int + charset string +) + +const ( + NUmStr = "0123456789" + CharStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + SpecStr = "+=-@#~,.[]()!%^*$" +) + +func parseArgs(lens int) { + length = lens + charset = "advance" + flag.Parse() +} + +func generatePasswd() string { + var passwd []byte = make([]byte, length, length) + var sourceStr string + if charset == "num" { + sourceStr = NUmStr + } else if charset == "char" { + sourceStr = charset + } else if charset == "mix" { + sourceStr = fmt.Sprintf("%s%s", NUmStr, CharStr) + } else if charset == "advance" { + sourceStr = fmt.Sprintf("%s%s%s", NUmStr, CharStr, SpecStr) + } else { + sourceStr = NUmStr + } + fmt.Println("source:", sourceStr) + for i := 0; i < length; i++ { + index := rand.Intn(len(sourceStr)) + passwd[i] = sourceStr[index] + } + return string(passwd) +} + +//GenPrivKey Generate private key +func GenPrivKey(lens int) string { + rand.Seed(time.Now().UnixNano()) + parseArgs(lens) + passwd := generatePasswd() + fmt.Println(passwd) + fmt.Printf("length:%d charset:%s\n", length, charset) + return passwd +} + +type Claims struct { + username string + password string + jwt.StandardClaims +} + +func setting(jwtkey []byte, username, password string) (string, error) { + expireTime := time.Now().Add(7 * 24 * time.Hour) + claims := &Claims{ + username: username, + password: password, + StandardClaims: jwt.StandardClaims{ + ExpiresAt: expireTime.Unix(), //expire date + IssuedAt: time.Now().Unix(), + Issuer: "127.0.0.1", // Signature issuer + Subject: "user token", //Signature subject + }, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + tokenString, err := token.SignedString(jwtkey) + if err != nil { + logs.Error(err) + return "", err + } + return tokenString, nil +} + +//GenToken Generate Token +func GenToken(username, password string) (string, error) { + pKey := GenPrivKey(16) + var jwtkey = []byte(pKey) + tokens, err := setting(jwtkey, username, password) + return tokens, err +} + +////解析token +//func getting(tokenString string) (string, struct{}){ +// token, claims, err := ParseToken(tokenString) +// if err != nil || !token.Valid { +// return "", struct{}{} +// } +// return token, +//} +// +//func ParseToken(tokenString string) (*jwt.Token, *Claims, error) { +// Claims := &Claims{} +// token, err := jwt.ParseWithClaims(tokenString, Claims, func(token *jwt.Token) (i interface{}, err error) { +// return jwtkey, nil +// }) +// return token, Claims, err +//} + +//EncryptMd5 encrypt md5 +func EncryptMd5(str string) string { + if str == ""{ + return str + } + sum := md5.Sum([]byte(str)) + return fmt.Sprintf("%x",sum) +} + diff --git a/cve-vulner-manager/common/common.go b/cve-vulner-manager/common/common.go new file mode 100644 index 0000000000000000000000000000000000000000..bafe45eae7ee21ad3244d2dcb336efc91bc78dd8 --- /dev/null +++ b/cve-vulner-manager/common/common.go @@ -0,0 +1,530 @@ +package common + +import ( + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "os" + "reflect" + "regexp" + "time" +) + +const DATE_FORMAT = "2006-01-02 15:04:05" + +func DesString(dbpwd string) (strs string) { + defer Catchs() + if "" != dbpwd || len(dbpwd) > 0 { + key := beego.AppConfig.String("key") + key1 := []byte(key) + bytes, _ := DePwdCode(dbpwd, key1) + strs = string(bytes) + } else { + strs = "" + } + return strs +} + +func AesString(dbpwd string) (strs string) { + defer Catchs() + pwd := []byte{} + if dbpwd == "" { + pwd = []byte("123") + } else { + pwd = []byte(dbpwd) + } + key := []byte(beego.AppConfig.String("key")) + strs, err := EnPwdCode(pwd, key) + if err != nil { + logs.Error(err) + } else { + logs.Info(strs) + } + return strs +} + +func GetTokenExpirTime() (token_expir_time int) { + token_expir_time, err := beego.AppConfig.Int("token_expir_time") + if err == nil { + return token_expir_time + } + return 3 +} + +func GetCurTime() string { + return time.Now().Format(DATE_FORMAT) +} + +func GetCurDate() string { + return time.Now().Format("2006-01-02") +} +func GetCurDateFormat() string { + return time.Now().Format("2006/01/02") +} + +func GetBeforeTime(days int) string { + nowTime := time.Now() + getTime := nowTime.AddDate(0, 0, days) + resTime := getTime.Format(DATE_FORMAT) // The format of the obtained time + logs.Info("Get the number of days：", days, ",Time ago：", resTime) + return resTime +} + +func GetSpecifiedTime(startTime time.Time, days int, localFlag bool) string { + getTime := startTime.AddDate(0, 0, days) + resTime := "" + if !localFlag { + h, _ := time.ParseDuration("1h") + resTime = getTime.Add(8 * h).Format(DATE_FORMAT) + } else { + resTime = getTime.Format(DATE_FORMAT) // The format of the obtained time + } + logs.Info("Get the number of days：", days, ",Time：", resTime) + return resTime +} + +func Catchs() { + if err := recover(); err != nil { + logs.Error("The program is abnormal, err: ", err) + } +} + +// Time string to timestamp +func PraseTimeInt(stringTime time.Time) int64 { + timeStr := stringTime.String() + if timeStr != "" && len(timeStr) > 19 { + timeStr = timeStr[:19] + } + loc, _ := time.LoadLocation("Local") + theTime, err := time.ParseInLocation(DATE_FORMAT, timeStr, loc) + if err == nil { + unixTime := theTime.Unix() + return unixTime + } else { + logs.Error(err) + } + return 0 +} + +// Get current timestamp +func CurTimestamp() int64 { + timeStamp := time.Now().Unix() + return timeStamp +} + +// Does the string slice contain a certain value +func IsValueInList(value string, list []string) bool { + for _, v := range list { + if v == value { + return true + } + } + return false +} + +func GetLocalCurTime() string { + localTime := time.Unix(time.Now().Unix(), 0) + local, _ := time.LoadLocation("Asia/Shanghai") + timeFormat := DATE_FORMAT + return localTime.In(local).Format(timeFormat) +} + +func TimeConverStr(timeStr string) string { + times, _ := time.Parse(DATE_FORMAT, timeStr) + timeUnix := times.Unix() + timx := time.Unix(timeUnix, 0).Format(DATE_FORMAT) + return timx +} + +func GetSpecialDate(beforeDate int) string { + year, month, date := time.Now().Date() + thisMonth := time.Date(year, month, beforeDate, 0, 0, 0, 0, time.Local) + var startDate time.Time + var curTime string + if date > beforeDate { + startDate = thisMonth.AddDate(0, 0, 0) + } else { + startDate = thisMonth.AddDate(0, -1, 0) + } + h, _ := time.ParseDuration("-1h") + curTime = startDate.Add(8 * h).Format(DATE_FORMAT) + return curTime +} + +func GetBeforeDate(flag int, value int) string { + getDate := "" + // flag: 0: current date; 1: previous day's date; 2: money one month's date; 3: previous year's date + now := time.Now() + switch flag { + case 0: + curDate := now.Format(DATE_FORMAT) + if len(curDate) > 1 { + getDate = curDate + } + case 1: + beforeDay := now.AddDate(0, 0, value) + getDate = beforeDay.Format(DATE_FORMAT) + case 2: + beforeMonth := now.AddDate(0, value, 0) + getDate = beforeMonth.Format(DATE_FORMAT) + case 3: + beforeYear := now.AddDate(value, 0, 0) + getDate = beforeYear.Format(DATE_FORMAT) + } + logs.Info("getDate: ", getDate) + return getDate +} + +func DeletePreAndSufSpace(str string) string { + strList := []byte(str) + spaceCount, count := 0, len(strList) + for i := 0; i <= len(strList)-1; i++ { + if strList[i] == 32 { + spaceCount++ + } else { + break + } + } + strList = strList[spaceCount:] + spaceCount, count = 0, len(strList) + for i := count - 1; i >= 0; i-- { + if strList[i] == 32 { + spaceCount++ + } else { + break + } + } + return string(strList[:count-spaceCount]) +} + +type CveDescription struct { + EnDesc string `json:"en"` + ZhDesc string `json:"zh"` +} + +type NodeCpe struct { + Cpe23Uri string `json:"cpe23Uri"` + CpeMatchString string `json:"cpeMatchString"` + Vulnerable string `json:"vulnerable"` +} + +type ConfNodes struct { + Operator string `json:"operator"` + Cpe []NodeCpe `json:"cpe"` +} + +type CveConfigurations struct { + Nodes []ConfNodes `json:"nodes"` +} + +type BmCvssV3 struct { + VectorString string `json:"vectorString"` + AttackComplexity string `json:"attackComplexity"` + AttackVector string `json:"attackVector"` + AvailabilityImpact string `json:"availabilityImpact"` + BaseSeverity string `json:"baseSeverity"` + UserInteraction string `json:"userInteraction"` + BaseScore float64 `json:"baseScore"` + PrivilegesRequired string `json:"privilegesRequired"` + Version string `json:"version"` + ConfidentialityImpact string `json:"confidentialityImpact"` + IntegrityImpact string `json:"integrityImpact"` + Scope string `json:"scope"` +} + +type ImBaseMetricV3 struct { + CvssV3 BmCvssV3 `json:"cvssV3"` + ImpactScore float64 `json:"impactScore"` + ExploitabilityScore float64 `json:"exploitabilityScore"` +} + +type BmCvssV2 struct { + VectorString string `json:"vectorString"` + AccessComplexity string `json:"accessComplexity"` + AvailabilityImpact string `json:"availabilityImpact"` + Authentication string `json:"authentication"` + Version string `json:"version"` + BaseScore float64 `json:"baseScore"` + IntegrityImpact string `json:"integrityImpact"` + ConfidentialityImpact string `json:"confidentialityImpact"` + AccessVector string `json:"accessVector"` +} + +type ImBaseMetricV2 struct { + AcInsufInfo string `json:"acInsufInfo"` + CvssV2 BmCvssV2 `json:"cvssV2"` + UserInteractionRequired string `json:"userInteractionRequired"` + Severity string `json:"severity"` + ObtainUserPrivilege string `json:"obtainUserPrivilege"` + ObtainAllPrivilege string `json:"obtainAllPrivilege"` + ImpactScore float64 `json:"impactScore"` + ExploitabilityScore float64 `json:"exploitabilityScore"` + ObtainOtherPrivilege string `json:"obtainOtherPrivilege"` +} + +type CveImpact struct { + BaseMetricV3 ImBaseMetricV3 `jsong:"baseMetricV3"` + BaseMetricV2 ImBaseMetricV2 `jsong:"baseMetricV2"` +} + +type CvePoc struct { + Source string `json:"source"` + Date string `json:"date"` + Path string `json:"path"` + Dbindex string `json:"dbindex"` + Url string `json:"url"` + Desc string `json:"desc"` +} + +type CveEvent struct { + Title string `json:"title"` + Date string `json:"date"` + Description string `json:"description"` + Url string `json:"url"` +} + +type CveReferenceData struct { + Url string `json:"url"` + Name string `json:"name"` + Refsource string `json:"refsource"` + Tags []string `json:"tags"` +} + +type CveVulType struct { + Cwe string `json:"cwe"` + En string `json:"en"` + Zh string `json:"zh"` +} + +type FixReferences struct { + Url string `json:"url"` + Refsource string `json:"refsource"` + Name string `json:"name"` + Tags []string `json:"tags"` +} + +type CveFixSuggest struct { + Detail string `jsong:"detail"` + References []FixReferences `jsong:"references"` +} + +type CveOriginData struct { + Ids string `json:"ids"` + CveNum string `json:"cveNum"` + UpdateType string `json:"updateType"` + CvePackName []string `json:"cvePackName"` + PackName []string `json:"packName"` + Description CveDescription `json:"description"` + Title string `json:"title"` + AffectProduct []string `json:"affectProduct"` + Configurations CveConfigurations `json:"configurations"` + CnnvdID string `json:"cnnvdID"` + CnvdID string `json:"cnvdID"` + PublishedDate string `json:"publishedDate"` + GetTime string `json:"getTime"` + EndGetTime string `json:"endGetTime"` + Impact CveImpact `json:"impact"` + VulStatus string `json:"vulStatus"` + Poc CvePoc `json:"poc"` + Event CveEvent `json:"event"` + ReferenceData []CveReferenceData `json:"referenceData"` + VulType CveVulType `json:"vulType"` + FixSuggest CveFixSuggest `json:"fixSuggest"` + Version string `json:"version"` + Credibility int `json:"credibility"` +} + +type UploadData struct { + Token string `json:"Token"` + CveData []CveOriginData +} + +type CveOriginDetailData struct { + CveNum string `json:"cveNum"` + CvePackName string `json:"cvePackName"` + Description CveDescription `json:"description"` + Title string `json:"title"` + Configurations CveConfigurations `json:"configurations"` + CnnvdID string `json:"cnnvdID"` + CnvdID string `json:"cnvdID"` + PublishedDate string `json:"publishedDate"` + GetTime string `json:"getTime"` + Impact CveImpact `json:"impact"` + VulStatus string `json:"vulStatus"` + Poc CvePoc `json:"poc"` + Event CveEvent `json:"event"` + ReferenceData []CveReferenceData `json:"referenceData"` + VulType CveVulType `json:"vulType"` + FixSuggest CveFixSuggest `json:"fixSuggest"` +} + +func GetRepoOrg() (string, error) { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return "", err + } + owner := BConfig.String("gitee::owner") + if owner == "" { + logs.Error("config gitee::owner error: invalid value is ", owner) + return "", errors.New("value is nil") + } + return owner, nil +} + +func CompareSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + + if (a == nil) != (b == nil) { + return false + } + bFlag := false + for _, av := range a { + for _, bv := range b { + if av == bv { + bFlag = true + break + } else { + bFlag = false + } + } + } + return bFlag +} + +func CreateDir(dir string) error { + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + os.Mkdir(dir, 0777) + } + } + return err +} + +func CreateAllDir(dir string) error { + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + os.MkdirAll(dir, 0777) + } + } + return err +} + +func StrFirstToUpper(str string) string { + var upperStr string + vv := []rune(str) + for i := 0; i < len(vv); i++ { + if i == 0 { + vv[i] -= 32 + upperStr += string(vv[i]) // + string(vv[i+1]) + } else { + upperStr += string(vv[i]) + } + } + return upperStr +} + +func BranchVersionRep(text string) string { + reg := regexp.MustCompile(`(\(.*\))|(（.*）)`) + return reg.ReplaceAllString(text, "") +} + +func StripStrReg(text string) string { + reg := regexp.MustCompile(`^[a-zA-Z]`) + return reg.ReplaceAllString(text, "") +} + +func FindSliceEm(slice []string, val string) (int, bool) { + if len(slice) == 0 { + return -1, false + } + for i, item := range slice { + if item == val { + return i, true + } + } + return -1, false +} + +func GetBeforeHourTime(hours time.Duration) string { + now := time.Now() + h, _ := time.ParseDuration("-1h") + beforeHours := now.Add(hours * h).Format(DATE_FORMAT) + return beforeHours +} + +func SliceRemoveDup(req interface{}) (ret []interface{}) { + if reflect.TypeOf(req).Kind() != reflect.Slice { + return + } + value := reflect.ValueOf(req) + for i := 0; i < value.Len(); i++ { + if i > 0 && reflect.DeepEqual(value.Index(i - 1).Interface(), value.Index(i).Interface()) { + continue + } + ret = append(ret, value.Index(i).Interface()) + } + return +} + +func IsContainsLetters(str string) bool { + if len(str) > 0 { + if isOk, _ := regexp.MatchString("[a-zA-Z-_]+", str); isOk { + return true + } + } else { + return true + } + return false +} + +func IsContainSpecialChar(str string, flag int8) bool { + if len(str) > 0 { + if flag == 1 { + if isOk, _ := regexp.MatchString("^[r|R][0-9]+(.)*[0-9]$", str); isOk { + return true + } + } else { + if isOk, _ := regexp.MatchString("^[v|V][0-9]+(.)*[0-9]$", str); isOk { + return true + } + } + } else { + return true + } + return false +} + +// Obtain token data from different communities and extract them into public data +func GetOwnerAndToken(organizationID int8) (owner, accessToken string) { + if organizationID == 2 { + owner = beego.AppConfig.String("opengauss::gauss_owner") + accessToken = beego.AppConfig.String("opengauss::git_gauss_token") + } else if organizationID == 3 { + owner = beego.AppConfig.String("mindspore::mindspore_owner") + accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") + } else if organizationID == 4 { + owner = beego.AppConfig.String("openlookeng::openlookeng_owner") + accessToken = beego.AppConfig.String("openlookeng::git_openlookeng_token") + } else { + owner = beego.AppConfig.String("gitee::owner") + accessToken = beego.AppConfig.String("gitee::git_token") + } + return +} + +func DeleteSliceValue(org []string, value string) (repSlice []string) { + repSlice = org + for k, v := range org { + if v == value { + repSlice = append(org[:k], org[k+1:]...) + break + } + } + return +} diff --git a/cve-vulner-manager/common/global.go b/cve-vulner-manager/common/global.go new file mode 100644 index 0000000000000000000000000000000000000000..47f424d13ad5c71b1bb4d96cc40f8270919a6c4c --- /dev/null +++ b/cve-vulner-manager/common/global.go @@ -0,0 +1,30 @@ +package common + +import ( + "os" +) + +type GlobalVal struct { + Dbpwd string + ClientID string + ClientSecret string + GitPassword string + HookPwd string + GitToken string + AesKey string + LoginKey string + DbUser string +} + +func InitGlobal() { + var gVal GlobalVal + gVal.Dbpwd = os.Getenv("DB_PWD") + gVal.DbUser = os.Getenv("DB_USER") + gVal.AesKey = os.Getenv("AES_KEY") + gVal.ClientID = os.Getenv("GITEE_CLIENT_ID") + gVal.ClientSecret = os.Getenv("GITEE_CLIENT_SECRET") + gVal.GitPassword = os.Getenv("GITEE_PASSWORD") + gVal.HookPwd = os.Getenv("HOOK_PWD") + gVal.GitToken = os.Getenv("GITEE_TOKEN") + //fmt.Println("env globalval: ", gVal) +} \ No newline at end of file diff --git a/cve-vulner-manager/common/logs.go b/cve-vulner-manager/common/logs.go new file mode 100644 index 0000000000000000000000000000000000000000..5dcc0a4b3f86273f3312b5171416f2b4b678ec6e --- /dev/null +++ b/cve-vulner-manager/common/logs.go @@ -0,0 +1,47 @@ +package common + +import ( + "encoding/json" + "fmt" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" +) + +func InitLogger() (err error) { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + fmt.Println("config init error:", err) + return + } + maxlines, lerr := BConfig.Int64("log::maxlines") + if lerr != nil { + maxlines = 20000 + } + + logConf := make(map[string]interface{}) + logConf["filename"] = BConfig.String("log::log_path") + level, _ := BConfig.Int("log::log_level") + logConf["level"] = level + logConf["maxlines"] = maxlines + + confStr, err := json.Marshal(logConf) + if err != nil { + fmt.Println("marshal failed,err:", err) + return + } + err = logs.SetLogger(logs.AdapterFile, string(confStr)) + if err != nil { + fmt.Println("marshal failed,err:", err) + return + } + logs.SetLogFuncCall(true) + return +} + +func LogInit() { + err := InitLogger() + if err != nil { + fmt.Println(err) + } + fmt.Println("log init success !") +} diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf new file mode 100644 index 0000000000000000000000000000000000000000..f70de9a7cd394bd6846cd5b5c76d808cb9a069d6 --- /dev/null +++ b/cve-vulner-manager/conf/app.conf @@ -0,0 +1,283 @@ +appname = cvevulner +httpport = 80 +runmode = dev +autorender = false +copyrequestbody = true +EnableDocs = true +sqlconn = +# Login key agreed by both parties +key = "${AES_KEY||djS*@+8K9{J!ymk6}" +initdb = 1 +# Login key +loginkey = "{${LOGIN_KEY||djS*@+8K9{-!yo%64}" +# Token validity period, unit: day +token_expir_time=3 +# Analysis instructions +analysisCmd = "/analysis" +endCmd = "/done" +rejectCmd = "/reject" +approveCmd = "/approve" +closeCmd = "/close" +errorCmd = "/error" +autoPrCmd = "/find-patch" +feedBackCmd = "/report-patch" +getCveCmd = "/get-cve" +labelFixed = "CVE/FIXED" +labelUnFix = "CVE/UNFIXED" +labeUnaffected = "CVE/UNAFFECTED" +totallabel = "CVE/FIXED,CVE/UNFIXED,CVE/UNAFFECTED" +labeAbiChanged = "abi-changed" +fileDir = "download" +saFileDir = "download/sa" +gaussFileDir = "download/gauss" +# release package download url http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv +rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" +#communityName = "src-openeuler:1,opengauss:2,mindspore:3" +communityName = "cve-test:1,cve-gauss:2,cve-mindspore:3,open-loo-keng:4" + +[timedtask] +packageDir = "cve-timed-task/" +GwController = "2 0 * * 0-6" +RunTableTask = "@every 40m" +ToExcelController = "2 0 * * 0-6" +RunMappingTask = "30 7 * * 0-6" +RunDelFileTask = "30 9 * * 0-6" +RunIssueRecordTask = "30 7 * * 0-6" +RunWhitelistTask = "@every 2h" +RunPackageCommitterTask = "@every 1h" +SupplementCveTask = "@every 27m" +LongSupplementCveTask = "30 1 * * 0-6" +ParseOpenGaussYamlTask = "30 2 * * 0-6" +ParseMindsporeYamlTask = "30 3 * * 0-6" + + + + + +; [mysql] +; dbhost = "${DB_URI||127.0.0.1}" +; dbport = 3310 +; dbuser = "${DB_USER||cve}" +; dbpwd = "${DB_PWD||***}" +; dbname = oneline_cvevulner +; dbprefix = cve_ +; maxidle = 30 +; maxconn = 3000 + + +[mysql] +dbhost = "127.0.0.1" +dbport = 3306 +dbuser = "root" +dbpwd = "root" +dbname = cvevulner +dbprefix = cve_ +maxidle = 30 +maxconn = 3000 + +[log] +log_level = 7 +log_dir = ./logs +log_path = logs/cve.log +maxlines=25000 +maxsize=204800 + +[crontab] +ymalflag = 2 +getymal = 0 0 15 * * * +eulerymalflag = 2 +eulergetymal = 0 53 17 * * * +cveflag = 2 +getcve = 0 40 15 * * * +oricveflag = 2 +oricvecheck = 0 16 19 * * * +getissueflag = 2 +getissue = 0 57 17 * * * +issueflag = 2 +createissue = 0 55 10 * * * +emergissueflag = 2 +emergcreateissue = 0 */3 * * * * +test = 0/10 * * * * * +gittokenflag = 2 +issueoath = * * */20 * * * +genexcelflag = 2 +genexcel = 0 0/10 * * * * +days = -30 +prcnum = 50 +printlogflag = 2 +printlog = 0 0/20 * * * * +unlockflag = 2 +unlock = 0 30 19 * * * +hookflag = 2 +prochook = 0 34 17 * * * +createhookflag = 2 +createhook = 0 29 11 * * * +exceptflag = 2 +exceptcve = 0 3 19 * * * +gensaflag = 2 +gensa = 0 3 19 * * * +issuestatisticflag = 2 +issuestatistic = 0 19 17 * * * +cvecredflag = 2 +cvecredit = 0 53 14 * * * +urgenttaskflag = 2 +urgenttask = 0 51 9-20 * * * +seclinkflag = 2 +seclinktask = 0 55 17 * * * +checkissuedelflag = 2 +checkissuedel = 0 22 16 * * * +# Statistics of issues created in different communities +issuecommunityflag = 2 +issuecommunity = 0 50 15 * * * +# Check whether the issue label is reasonable +issuelabelcheckflag = 2 +issuelabelcheck = 0 */30 * * * * + + +[gitee] +#owner = cve-test +#owner = src-openeuler +#path = test +#email = 1499273991@qq.com +#redirect_uri = http://119.8.126.102:80/v1/issue/oauth/callback +# -------jianjun gitee 配置 -------- +#owner = src-openeuler +owner = cve-test +path = jasper +email = 7844966+zhangjianjun_code@user.noreply.gitee.com +redirect_uri = http://159.138.2.2:80/v1/issue/oauth/callback +owner_openuler= openeuler + + +scope = user_info projects pull_requests issues notes keys hook groups gists enterprises emails +# Obtain first from the system environment variables, use the default value if it fails **** +client_id = "${GITEE_CLIENT_ID||****}" +client_secret = "${GITEE_CLIENT_SECRET||****}" +password = "${GITEE_PASSWORD||****}" + +# git token +git_token = "${GITEE_TOKEN||xxx}" + +[hook] +hookpwd = "${HOOK_PWD||***}" +hookurl = http://159.138.2.2:80/v1/issue/hook/event +hook_timeout = 600 +# 1 create, 2 Not create +create_hook = 2 +# Whether to delete all webhooks: 1: do not delete; 2: delete +delete_hook = 2 + + +[yaml] +apiurl = https://api.openeuler.org/pkgmanage +#eulerurl = http://159.138.0.250:8081/v1/versions +eulerurl = http://omapi.osinfra.cn/v1/versions + +[cve] +cveref = https://nvd.nist.gov/vuln/detail/ +openeulernum = 3000 +# Import cve as data after 2018 +cve_number = 1990 +# Manual data import restrictions +cve_number_m = 1990 +# Tool import data restriction +cve_number_t = 1990 +# Create an issue's repo whitelist;1: open; 2: close +issue_whitelist = 2 +# List of affected branches： openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3 +affected_branchs = "openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3" +# Close the highest privilege of issue +close_issue_privilege = 2 +# abnormal cve status ,Use "," to separate multiple states +abn_cve_status =3 +# SA number, The remaining amount +sa_re_amount = 4000 +# The number of days the issue counts the mailing list +cve_statistics_date = 20 +# before 3 Month +before_month = -3 +# Trusted data query(0-3) +credibility_level = 3 +# Date before adding the security bulletin link task +sec_link_date = -100 +# Branch configuration of cve that needs to submit an issue +submit_issue_branch = "openEuler-20.03-LTS,openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3,openEuler-20.03-LTS-Next,openEuler-21.03,master" +# De-duplication +de_duplication_date = -300 +# table num +yaml_config_table = "cve_open_euler_repo_origin:1,cve_open_guss_yaml:2,cve_mind_spore_yaml:3,cve_open_lookeng_yaml:4" +bot_cu_account = "openeuler-ci-bot,opengauss-bot,mindspore-ci-bot,mindspore_ci,i-robot,CVE,I-am-a-robot" +out_of_sync_branch = "openEuler1.0-base,openEuler1.0" + +[reflink] +comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/cve-vulner-manager/doc/md/manual.md +gauss_comment_cmd = https://gitee.com/opengauss/security/blob/master/cve/manual.md +spore_comment_cmd = https://gitee.com/mindspore/community/blob/master/security/cve_issue_template.md +looKeng_comment_cmd = https://gitee.com/openlookeng/community/blob/master/security/cve/doc/md/manual.md +openeuler_web = https://www.openeuler.org +#openeuler_web = http://119.13.93.224:9090 +get_cve_link = http://127.0.0.1:8080/pulls/cve/info + +[excel] +forcerewrite = false +snprefix = op-2020-10- +snsuffix = 1002 +# Version package excel download address +# example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; +# openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv +# public sa format: openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https +v_pack_20_03_url = "openEuler-20.03-LTS-SP1@https;openEuler-20.03-LTS-SP2@https;openEuler-20.03-LTS-SP3@https" +# Time difference in different time zones +sa_timestamp_zone = 28800 +unaffect_year = 2018 + +[xml] +updateinfo_path = updateinfo.xml +cvrf_upload_path = http://159.138.2.2:9090/ +use_openeuler_num = openEuler-SA-2021-1001 + +[email] +email_name = "${EMAIL_NAME||***}" +email_pwd = "${EMAIL_PWD||***}" +email_host = mailman-exim4-service.mail.svc.cluster.local +email_port = 25 + +[opengauss] +#gauss_owner = opengauss +gauss_owner = cve-gauss +# git token +git_gauss_token = "${GITEE_GAUSS_TOKEN||xxx}" +gauss_version = 2.0.0,1.1.0,1.0.1,1.0.0 +gauss_issue_path = security +gauss_branch_path = openGauss-server +sa_init_value = 1001 +pr_repo = "openGauss-server,openGauss-connector-jdbc,openGauss-connector-odbc,openGauss-OM,openGauss-third_party" +nvd_relink = https://nvd.nist.gov/vuln/detail/ + + +[mindspore] +#mindspore_owner = mindspore +mindspore_owner = cve-mindspore +# git token +git_mindspore_token = "${GITEE_MINDSPORE_TOKEN||xxx}" +mindspore_version = "master,r1.2" + +[obs] +access_key_id = "${OBS_KEY_ID||xxx}" +secret_access_key = "${OBS_SECRET_KEY||xxx}" +endpoint = obs.ap-southeast-1.myhuaweicloud.com +bucket = openeuler-cve-cvrf +upload_cvrf_dir = "cve-manager-cvrf/" +download_cvrf_dir = "cvrf/" +upload_updateinfo_dir = "cve-manager-updateinfo/" + +[cveagency] +#url = "http://cve-manager-agent.cve-manager-agent.svc.cluster.local/v1/cve/track" +url = "http://127.0.0.1:8090/v1/cve/track" + +[openlookeng] +#openlookeng_owner = openlookeng +openlookeng_owner = open-loo-keng +# git token +git_openlookeng_token = "${GITEE_OPENLOOKENG_TOKEN||xxx}" +openlookeng_version = "master" \ No newline at end of file diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf new file mode 100644 index 0000000000000000000000000000000000000000..fff0ba38dfa4094cffebbadc513bbf63894775f9 --- /dev/null +++ b/cve-vulner-manager/conf/product_app.conf @@ -0,0 +1,269 @@ +appname = cve-vulner-manager +httpport = 8080 +runmode = prod +autorender = false +copyrequestbody = true +EnableDocs = true +sqlconn = +# Login key agreed by both parties +key = "${AES_KEY||djS*@+8K9{J!ymk6}" +initdb = 1 +# Login key +loginkey = "{${LOGIN_KEY||djS*@+8K9{-!yo%64}" +# Token validity period, unit: day +token_expir_time=3 +# Analysis instructions +analysisCmd = "/analysis" +endCmd = "/done" +rejectCmd = "/reject" +approveCmd = "/approve" +closeCmd = "/close" +errorCmd = "/error" +autoPrCmd = "/find-patch" +feedBackCmd = "/report-patch" +getCveCmd = "/get-cve" +labelFixed = "CVE/FIXED" +labelUnFix = "CVE/UNFIXED" +labeUnaffected = "CVE/UNAFFECTED" +totallabel = "CVE/FIXED,CVE/UNFIXED,CVE/UNAFFECTED" +labeAbiChanged = "abi-changed" +fileDir = "download" +saFileDir = "download/sa" +gaussFileDir = "download/gauss" +# release package download url +rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" +communityName = "src-openeuler:1,opengauss:2,mindspore:3,openlookeng:4" + + + +[timedtask] +packageDir = "cve-timed-task/" +GwController = "2 0 * * 0-6" +RunTableTask = "@every 40m" +ToExcelController = "2 0 * * 0-6" +RunMappingTask = "30 7 * * 0-6" +RunDelFileTask = "30 9 * * 0-6" +RunIssueRecordTask = "30 7 * * 0-6" +RunWhitelistTask = "@every 2h" +RunPackageCommitterTask = "@every 1h" +SupplementCveTask = "@every 27m" +LongSupplementCveTask = "30 1 * * 0-6" +ParseOpenGaussYamlTask = "30 2 * * 0-6" +ParseMindsporeYamlTask = "30 3 * * 0-6" + +; [mysql] +; dbhost = "${DB_URI||***}" +; dbport = 3306 +; dbuser = "${DB_USER||cve}" +; dbpwd = "${DB_PWD||***}" +; dbname = cvevulner +; dbprefix = cve_ +; maxidle = 30 +; maxconn = 3000 + + +[mysql] +dbhost = "127.0.0.1" +dbport = 3306 +dbuser = "root" +dbpwd = "root" +dbname = cvevulner +dbprefix = cve_ +maxidle = 30 +maxconn = 3000 + + +[log] +log_level = 5 +log_dir = ./logs +log_path = logs/cve.log +maxlines=20000 +maxsize=204800 + +[crontab] +ymalflag = 1 +getymal = 0 40 1 * * * +eulerymalflag = 1 +eulergetymal = 0 0 1 * * * +cveflag = 1 +getcve = 0 */10 * * * * +oricveflag = 1 +oricvecheck = 0 0 2 * * * +getissueflag = 1 +getissue = 0 20 1,12 * * * +issueflag = 1 +createissue = 0 0 6 * * * +emergissueflag = 1 +emergcreateissue = 0 */10 * * * * +test = 0/10 * * * * * +gittokenflag = 2 +issueoath = * * */20 * * * +genexcelflag = 1 +genexcel = 0 30 7 * * * +days = -30 +prcnum = 50 +printlogflag = 1 +printlog = 0 0 1 * * 2,4,6 +unlockflag = 1 +unlock = 0 */5 * * * * +hookflag = 1 +prochook = 0 1 8 * * * +createhookflag = 2 +createhook = 0 20 2 * * * +exceptflag = 2 +exceptcve = 0 0 4 * * * +gensaflag = 1 +gensa = 0 1 0 * * * +issuestatisticflag = 1 +issuestatistic = 0 1 10 * * 1,3,5 +cvecredflag = 1 +cvecredit = 0 0 5 * * * +urgenttaskflag = 1 +urgenttask = 0 0 9-20 * * * +seclinkflag = 1 +seclinktask = 0 0 9,12,18 * * * +checkissuedelflag = 1 +checkissuedel = 0 30 3 * * 1,3,5 +# Statistics of issues created in different communities +issuecommunityflag = 1 +issuecommunity = 0 0 */6 * * * +# Check whether the issue label is reasonable +issuelabelcheckflag = 1 +issuelabelcheck = 0 30 2 * * * + + +[gitee] +owner = src-openeuler +path = jasper +#email = 7844966+zhangjianjun_code@user.noreply.gitee.com +email = ***** +#redirect_uri = https://api.openeuler.org/cve-manager/v1/issue/oauth/callback + +scope = user_info projects pull_requests issues notes keys hook groups gists enterprises emails +# Obtain first from the system environment variables, use the default value if it fails **** +client_id = "${GITEE_CLIENT_ID||****}" +client_secret = "${GITEE_CLIENT_SECRET||****}" +password = "${GITEE_PASSWORD||****}" + +# git token +git_token = "${GITEE_TOKEN||xxx}" +# openeuler owner +owner_openuler= openeuler + + +[hook] +hookpwd = "${HOOK_PWD||***}" +hookurl = https://api.openeuler.org/cve-manager/v1/issue/hook/event +hook_timeout = 600 +# 1 create, 2 Not create +create_hook = 2 +# Whether to delete all webhooks: 1: do not delete; 2: delete +delete_hook = 2 + +[yaml] +apiurl = https://api.openeuler.org/pkgmanage +eulerurl = http://omapi.osinfra.cn/v1/versions + +[cve] +cveref = https://nvd.nist.gov/vuln/detail/ +openeulernum = 3000 +# Import cve as data after 2018 +cve_number = 1990 +# Manual data import restrictions +cve_number_m = 1990 +# Tool import data restriction +cve_number_t = 2018 +# Create an issue's repo whitelist;1: open; 2: close +issue_whitelist = 2 +# List of affected branches: openEuler-20.03-LTS,openEuler-20.03-LTS-SP1 +affected_branchs = "openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3" +# Close the highest privilege of issue:1:open;2:close +close_issue_privilege = 2 +# abnormal cve status ,Use "," to separate multiple states +abn_cve_status =3 +# SA number, The remaining amount +sa_re_amount = 4000 +# The number of days the issue counts the mailing list +cve_statistics_date = 20 +# before 3 Month +before_month = -3 +# Trusted data query(0-3) +credibility_level = 3 +# Date before adding the security bulletin link task +sec_link_date = -100 +# Branch configuration of cve that needs to submit an issue +submit_issue_branch = "openEuler-20.03-LTS,openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3,openEuler-20.03-LTS-Next,openEuler-21.03,master" +# De-duplication +de_duplication_date = -100 +# table num +yaml_config_table = "cve_open_euler_repo_origin:1,cve_open_guss_yaml:2,cve_mind_spore_yaml:3,cve_open_lookeng_yaml:4" +bot_cu_account = "openeuler-ci-bot,opengauss-bot,mindspore-ci-bot,mindspore_ci,i-robot,CVE,I-am-a-robot" +out_of_sync_branch = "openEuler1.0-base,openEuler1.0" + +[reflink] +comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/cve-vulner-manager/doc/md/manual.md +gauss_comment_cmd = https://gitee.com/opengauss/security/blob/master/cve/manual.md +spore_comment_cmd = https://gitee.com/mindspore/community/blob/master/security/cve_issue_template.md +looKeng_comment_cmd = https://gitee.com/openlookeng/community/blob/master/security/cve/doc/md/manual.md +openeuler_web = https://www.openeuler.org +get_cve_link = http://cve-manager-analysis-service.cve-manager.svc.cluster.local:8080/pulls/cve/info + +[excel] +forcerewrite = false +snprefix = op-2020-10- +snsuffix = 1002 +# Version package excel download address +# example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; +# openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv +# public sa format: openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https +v_pack_20_03_url = "openEuler-20.03-LTS-SP1@https;openEuler-20.03-LTS-SP2@https;openEuler-20.03-LTS-SP3@https" +# Time difference in different time zones +sa_timestamp_zone = 28810 +unaffect_year = 2018 + +[xml] +updateinfo_path = updateinfo.xml +cvrf_upload_path = http://159.138.2.2:9090/ +use_openeuler_num = openEuler-SA-2021-1001,openEuler-SA-2021-1002 + +[email] +email_name = "${EMAIL_NAME||***}" +email_pwd = "${EMAIL_PWD||***}" +email_host = mailman-exim4-service.mail.svc.cluster.local +email_port = 25 + +[opengauss] +gauss_owner = opengauss +# git token +git_gauss_token = "${GITEE_GAUSS_TOKEN||xxx}" +gauss_version = "master" +gauss_issue_path = security +gauss_branch_path = openGauss-server +sa_init_value = 1001 +pr_repo = "openGauss-server,openGauss-connector-jdbc,openGauss-connector-odbc,openGauss-OM,openGauss-third_party" +nvd_relink = https://nvd.nist.gov/vuln/detail/ + + +[mindspore] +mindspore_owner = mindspore +# git token +git_mindspore_token = "${GITEE_MINDSPORE_TOKEN||xxx}" +mindspore_version = "master" + +[obs] +access_key_id = "${OBS_KEY_ID||xxx}" +secret_access_key = "${OBS_SECRET_KEY||xxx}" +endpoint = obs.ap-southeast-1.myhuaweicloud.com +bucket = openeuler-cve-cvrf +upload_cvrf_dir = "cve-manager-cvrf/" +download_cvrf_dir = "cvrf/" +upload_updateinfo_dir = "cve-manager-updateinfo/" + +[cveagency] +url = "http://cve-manager-agent.cve-manager-agent.svc.cluster.local/v1/cve/track" + +[openlookeng] +openlookeng_owner = openlookeng +# git token +git_openlookeng_token = "${GITEE_OPENLOOKENG_TOKEN||xxx}" +openlookeng_version = "master" \ No newline at end of file diff --git a/cve-vulner-manager/controllers/cve.go b/cve-vulner-manager/controllers/cve.go new file mode 100644 index 0000000000000000000000000000000000000000..524ba599b76510f691d2b3ebe99300f30dc8b64f --- /dev/null +++ b/cve-vulner-manager/controllers/cve.go @@ -0,0 +1,327 @@ +package controllers + +import ( + "cvevulner/common" + "cvevulner/errcode" + "cvevulner/models" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "time" +) + +type CveIssueInfoController struct { + beego.Controller +} + +func (c *CveIssueInfoController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type CveIssueInfoData struct { + IssueNum string `json:"issue_id"` + CveNum string `json:"CVE_num"` + OpeneulerScore float64 `json:"openeuler_score"` + NvdScore float64 `json:"NVD_score"` + Branch string `json:"milestone"` +} + +// @Title Get cveissueinfo +// @Description get cve +// @Param status int true (0,1,2) +// @Success 200 {object} CveIssueInfoData +// @Failure 403 :status is err +// @router / [get] +func (u *CveIssueInfoController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var ird []CveIssueInfoData + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + resp["body"] = []CveIssueInfoData{} + resp["total"] = 0 + //defer u.RetData(resp) + milestone := u.GetString("milestone", "") + count := models.QueryIssueCount(milestone) + if count > 0 { + resp["total"] = count + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + currentPage, err := u.GetInt("currentPage", 1) + if err != nil { + logs.Error("err: ", err, ", currentPage: ", currentPage) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + pageSize, err := u.GetInt("pageSize", 100) + if err != nil { + logs.Error("err: ", err, ", pageSize: ", pageSize) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + issueData, issueErr := models.QueryCveIssueByBranch(currentPage, pageSize, milestone) + if issueErr == nil && len(issueData) > 0 { + for _, issues := range issueData { + var irda CveIssueInfoData + irda.CveNum = issues.CveNum + irda.IssueNum = issues.IssueNum + irda.OpeneulerScore = issues.OpeneulerScore + irda.NvdScore = issues.NvdScore + irda.Branch = issues.AffectProduct + ird = append(ird, irda) + } + resp["body"] = ird + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetData(resp) + } else { + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } + } +} + +type CveAllIssueController struct { + beego.Controller +} + +func (c *CveAllIssueController) RetCveData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type CveAllIssueoData struct { + IssueNum string `json:"issue_id"` + CveNum string `json:"CVE_num"` + OpeneulerScore float64 `json:"openeuler_score"` + NvdScore float64 `json:"NVD_score"` + Branch string `json:"milestone"` + CveLevel string `json:"CVE_level"` + PlanClosedTime string `json:"plan_closed_time"` + CvePublicTime string `json:"CVE_public_time"` + CveVtopicPublicTime string `json:"CVE_vtopic_public_time"` + CveVtopicRecTime string `json:"CVE_vtopic_rec_time"` + RpmPublicTime string `json:"rpm_public_time"` + SaPublicTime string `json:"SA_public_time"` +} + +// @Title Obtain all cve data and provide operation kanban for use +// @Description get issue +// @Param pageSize 1,currentPage 10, updatetime nil +// @Success 200 {object} CveAllIssueoData +// @Failure 403 :status is err +// @router / [get] +func (u *CveAllIssueController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var cid []CveAllIssueoData + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + resp["body"] = []CveAllIssueoData{} + resp["total"] = 0 + //defer u.RetCveData(resp) + communityFlag, flagErr := u.GetInt("communityFlag", 0) + if flagErr != nil { + communityFlag = 0 + } + startTime := u.GetString("startTime", "") + count := models.QueryCveAllIssueCount(communityFlag, startTime) + if count > 0 { + resp["total"] = count + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + currentPage, err := u.GetInt("currentPage", 1) + if err != nil { + logs.Error("err: ", err, ", currentPage: ", currentPage) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetCveData(resp) + return + } + pageSize, err := u.GetInt("pageSize", 100) + if err != nil { + logs.Error("err: ", err, ", pageSize: ", pageSize) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetCveData(resp) + return + } + issueData, issueErr := models.QueryCveAllIssueData(currentPage, pageSize, communityFlag, startTime) + if issueErr == nil && len(issueData) > 0 { + for _, issues := range issueData { + var irda CveAllIssueoData + irda.CveNum = issues.CveNum + irda.IssueNum = issues.IssueNum + irda.OpeneulerScore = issues.OpeneulerScore + irda.NvdScore = issues.NvdScore + irda.Branch = issues.AffectedVersion + irda.CveLevel = issues.CveLevel + irda.CveVtopicPublicTime = issues.FirstPerTime + irda.CveVtopicRecTime = issues.FirstGetTime + irda.PlanClosedTime = CvePlanCloseTime(issues.CreateTime, issues.CveLevel) + if len(issues.CpublicTime) > 10 { + irda.CvePublicTime = issues.CpublicTime[:10] + } else { + irda.CvePublicTime = issues.CpublicTime + } + if len(issues.RpmReleaseTime) > 2 { + irda.RpmPublicTime = issues.RpmReleaseTime + } else { + irda.RpmPublicTime = issues.SaReleaseTime + } + irda.SaPublicTime = issues.SaReleaseTime + cid = append(cid, irda) + } + resp["body"] = cid + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetCveData(resp) + } else { + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetCveData(resp) + return + } + } +} + +func CvePlanCloseTime(startTime time.Time, cveLevel string) string { + endLocal := "" + switch cveLevel { + case "Critical": + endLocal = common.GetSpecifiedTime(startTime, 7, false) + case "High": + endLocal = common.GetSpecifiedTime(startTime, 14, false) + case "Medium", "Low": + endLocal = common.GetSpecifiedTime(startTime, 30, false) + default: + endLocal = common.GetSpecifiedTime(startTime, 60, false) + } + return endLocal +} + + +type KanbanCveAllIssueController struct { + beego.Controller +} + +func (c *KanbanCveAllIssueController) RetCveData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type KanbanCveAllIssueoData struct { + IssueNum string `json:"issue_id"` + CveNum string `json:"CVE_num"` + OpeneulerScore float64 `json:"openeuler_score"` + NvdScore float64 `json:"NVD_score"` + Branch string `json:"milestone"` + CveLevel string `json:"CVE_level"` + PlanClosedTime string `json:"plan_closed_time"` + CvePublicTime string `json:"CVE_public_time"` + CveVtopicPublicTime string `json:"CVE_vtopic_public_time"` + CveVtopicRecTime string `json:"CVE_vtopic_rec_time"` + RpmPublicTime string `json:"rpm_public_time"` + SaPublicTime string `json:"SA_public_time"` +} + +// @Title Obtain all cve data and provide operation kanban for use +// @Description get issue +// @Param pageSize 1,currentPage 10, updatetime nil +// @Success 200 {object} CveAllIssueoData +// @Failure 403 :status is err +// @router / [get] +func (u *KanbanCveAllIssueController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var cid []CveAllIssueoData + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + resp["body"] = []CveAllIssueoData{} + resp["total"] = 0 + //defer u.RetCveData(resp) + communityFlag, flagErr := u.GetInt("communityFlag", 0) + if flagErr != nil { + communityFlag = 0 + } + count := models.QueryCveOrgIssueCount(communityFlag) + if count > 0 { + resp["total"] = count + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + currentPage, err := u.GetInt("currentPage", 1) + if err != nil { + logs.Error("err: ", err, ", currentPage: ", currentPage) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetCveData(resp) + return + } + pageSize, err := u.GetInt("pageSize", 100) + if err != nil { + logs.Error("err: ", err, ", pageSize: ", pageSize) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetCveData(resp) + return + } + issueData := models.QueryCveOrgIssueData(currentPage, pageSize, communityFlag) + if len(issueData) > 0 { + for _, org := range issueData { + var irda CveAllIssueoData + irda.CveNum = org.CveNumber + irda.IssueNum = org.Number + issues, isErr := models.QueryCveSingleIssueData(communityFlag, org.RepoPath, org.CveNumber, org.Number) + if isErr != nil { + cid = append(cid, irda) + }else { + irda.OpeneulerScore = issues.OpeneulerScore + irda.NvdScore = issues.NvdScore + irda.Branch = issues.AffectedVersion + if issues.OpeneulerScore >= 0 { + irda.CveLevel = models.OpenEulerScoreProc(issues.OpeneulerScore) + } + irda.CveVtopicPublicTime = issues.FirstPerTime + irda.CveVtopicRecTime = issues.FirstGetTime + irda.PlanClosedTime = CvePlanCloseTime(issues.CreateTime, issues.CveLevel) + if len(issues.CpublicTime) > 10 { + irda.CvePublicTime = issues.CpublicTime[:10] + } else { + irda.CvePublicTime = issues.CpublicTime + } + if len(issues.RpmReleaseTime) > 2 { + irda.RpmPublicTime = issues.RpmReleaseTime + } else { + irda.RpmPublicTime = issues.SaReleaseTime + } + irda.SaPublicTime = issues.SaReleaseTime + cid = append(cid, irda) + } + } + resp["body"] = cid + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetCveData(resp) + } else { + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetCveData(resp) + return + } + } +} \ No newline at end of file diff --git a/cve-vulner-manager/controllers/cvedetail.go b/cve-vulner-manager/controllers/cvedetail.go new file mode 100644 index 0000000000000000000000000000000000000000..c7012f9346012b208ac74961d3a06314a2b92ab9 --- /dev/null +++ b/cve-vulner-manager/controllers/cvedetail.go @@ -0,0 +1,245 @@ +package controllers + +import ( + "cvevulner/common" + "cvevulner/errcode" + "cvevulner/models" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" +) + +// Operations about Packages + +type CveDetailController struct { + beego.Controller +} + +func (c *CveDetailController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title Get cvedetail +// @Description get cvedetail +// @Param cvenumber type string true +// @Success 200 {object} models.uploadcve +// @Failure 403 :cvenumber is err +// @router / [get] +func (u *CveDetailController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "The ip address requested by the client: ", addr, + ",Header: ", req.Header, ",body: ", req.Body) + resp := make(map[string]interface{}) + var cod common.CveOriginDetailData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = cod + //defer u.RetData(resp) + //Judge whether it is legal + token := u.GetString("token") + if token == "" { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + resp["body"] = []ResultData{} + logs.Error("token acquisition failed") + u.RetData(resp) + return + } else { + // Check token + ok := models.CheckToken(token) + if !ok { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + resp["body"] = []ResultData{} + logs.Error("token verification failed") + u.RetData(resp) + return + } + } + cveNum := u.GetString("cveNum") + if cveNum == "" { + logs.Error("cveNum, Parameter error") + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + cveType, typeError := u.GetInt64("cveType") + if typeError != nil || cveType == 0 { + logs.Error("cveType, Parameter error") + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + // The original data comes from the display of the Chinese Academy of Sciences + if cveType == 1 { + var ou models.OriginUpstream + ouErr := models.GetOriginUpstream(cveNum, &ou) + if ouErr != nil || ou.CveId == 0 { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } + cod.CveNum = ou.CveNum + cod.CvePackName = ou.PackName + cod.Title = ou.Title + cod.CnnvdID = ou.CnnvdID + cod.CnvdID = ou.CnvdID + cod.PublishedDate = ou.PublishedDate + cod.VulStatus = ou.VulStatus + cod.GetTime = ou.FirstPerTime + var oud models.OriginUpstreamDesc + oudErr := models.GetOriginDesc(ou.CveId, &oud) + if oudErr == nil && oud.DescId > 0 { + cod.Description.EnDesc = oud.EnDescription + cod.Description.ZhDesc = oud.ZhDescription + } + var ouc models.OriginUpstreamConfig + oucErr := models.GetOriginConfig(ou.CveId, &ouc) + if oucErr == nil && ouc.ConfId > 0 { + var oucn []models.OriginUpstreamConfigNode + var cf common.CveConfigurations + oucnNum, oucnErr := models.GetOriginConfigNode(ouc.ConfId, &oucn) + if oucnNum > 0 && oucnErr == nil { + for _, nodes := range oucn { + var cn common.ConfNodes + cn.Operator = nodes.Operator + var oucnc []models.OriginUpstreamConfigNodeCpe + oucncNum, oucncErr := models.GetOriginConfigNodeCpe(nodes.NodeId, &oucnc) + if oucncNum > 0 && oucncErr == nil { + for _, cpe := range oucnc { + var cnc common.NodeCpe + cnc.Vulnerable = cpe.Vulnerable + cnc.CpeMatchString = cpe.CpeMatchString + cnc.Cpe23Uri = cpe.Cpe23Uri + cn.Cpe = append(cn.Cpe, cnc) + } + } + cf.Nodes = append(cf.Nodes, cn) + } + } + cod.Configurations = cf + } + cveImpact, ok := models.QueryCveImpact(ou.CveId) + if ok && cveImpact.ImpactId > 0 { + cveScore, ok := models.QueryCveScore(cveImpact.ImpactId, "v3") + if ok && cveScore.ScoreId > 0 { + cveScV3, ok := models.QueryCveCvssV3(cveScore.ScoreId) + if ok && cveScV3.V3Id > 0 { + logs.Info(cveScV3) + cod.Impact.BaseMetricV3.CvssV3.Version = cveScV3.Version + cod.Impact.BaseMetricV3.CvssV3.VectorString = cveScV3.VectorString + cod.Impact.BaseMetricV3.CvssV3.AttackComplexity = cveScV3.AttackComplexity + cod.Impact.BaseMetricV3.CvssV3.AttackVector = cveScV3.AttackVector + cod.Impact.BaseMetricV3.CvssV3.AvailabilityImpact = cveScV3.AvailabilityImpact + cod.Impact.BaseMetricV3.CvssV3.BaseSeverity = cveScV3.BaseSeverity + cod.Impact.BaseMetricV3.CvssV3.UserInteraction = cveScV3.UserInteraction + cod.Impact.BaseMetricV3.CvssV3.BaseScore = cveScV3.BaseScore + cod.Impact.BaseMetricV3.CvssV3.PrivilegesRequired = cveScV3.PrivilegesRequired + cod.Impact.BaseMetricV3.CvssV3.ConfidentialityImpact = cveScV3.ConfidentialityImpact + cod.Impact.BaseMetricV3.CvssV3.IntegrityImpact = cveScV3.IntegrityImpact + cod.Impact.BaseMetricV3.CvssV3.Scope = cveScV3.Scope + cod.Impact.BaseMetricV3.ExploitabilityScore = cveScV3.ExploitabilityScore + cod.Impact.BaseMetricV3.ImpactScore = cveScV3.ImpactScore + } + } + cveScoreV2, ok2 := models.QueryCveScore(cveImpact.ImpactId, "v2") + if ok2 && cveScoreV2.ScoreId > 0 { + cveScV2, okV2 := models.QueryCveCvssV2(cveScoreV2.ScoreId) + if okV2 && cveScV2.V2Id > 0 { + logs.Info(cveScV2) + cod.Impact.BaseMetricV2.CvssV2.VectorString = cveScV2.VectorString + cod.Impact.BaseMetricV2.CvssV2.AccessComplexity = cveScV2.AccessComplexity + cod.Impact.BaseMetricV2.CvssV2.AvailabilityImpact = cveScV2.AvailabilityImpact + cod.Impact.BaseMetricV2.CvssV2.Authentication = cveScV2.Authentication + cod.Impact.BaseMetricV2.CvssV2.Version = cveScV2.Version + cod.Impact.BaseMetricV2.CvssV2.BaseScore = cveScV2.BaseScore + cod.Impact.BaseMetricV2.CvssV2.IntegrityImpact = cveScV2.IntegrityImpact + cod.Impact.BaseMetricV2.CvssV2.ConfidentialityImpact = cveScV2.ConfidentialityImpact + cod.Impact.BaseMetricV2.CvssV2.AccessVector = cveScV2.AccessVector + cod.Impact.BaseMetricV2.AcInsufInfo = cveScV2.AcInsufInfo + cod.Impact.BaseMetricV2.UserInteractionRequired = cveScV2.UserInteractionRequired + cod.Impact.BaseMetricV2.Severity = cveScV2.Severity + cod.Impact.BaseMetricV2.ObtainUserPrivilege = cveScV2.ObtainUserPrivilege + cod.Impact.BaseMetricV2.ObtainAllPrivilege = cveScV2.ObtainAllPrivilege + cod.Impact.BaseMetricV2.ImpactScore = cveScV2.ImpactScore + cod.Impact.BaseMetricV2.ExploitabilityScore = cveScV2.ExploitabilityScore + cod.Impact.BaseMetricV2.ObtainOtherPrivilege = cveScV2.ObtainOtherPrivilege + } + } + } + var oup models.OriginUpstreamPoc + oupErr := models.GetOriginPoc(ou.CveId, &oup) + if oupErr == nil && oup.PocId > 0 { + cod.Poc.Url = oup.Url + cod.Poc.Date = oup.Date + cod.Poc.Source = oup.Source + cod.Poc.Path = oup.Path + cod.Poc.Desc = oup.Desc + cod.Poc.Dbindex = oup.Dbindex + } + var oue models.OriginUpstreamEvent + oueErr := models.GetOriginEvent(ou.CveId, &oue) + if oueErr == nil && oue.EventId > 0 { + cod.Event.Date = oue.Date + cod.Event.Url = oue.Url + cod.Event.Description = oue.Description + cod.Event.Title = oue.Title + } + var our []models.OriginUpstreamReference + ourNum, ourErr := models.GetOriginReference(ou.CveId, &our) + if ourErr == nil && ourNum > 0 { + for _, reference := range our { + var crd common.CveReferenceData + crd.Url = reference.Url + crd.Name = reference.Name + crd.Tags = append(crd.Tags, reference.Tags) + crd.Refsource = reference.Refsource + cod.ReferenceData = append(cod.ReferenceData, crd) + } + } + var ouv models.OriginUpstreamVulType + ouvErr := models.GetOriginVulType(ou.CveId, &ouv) + if ouvErr == nil && ouv.VulId > 0 { + cod.VulType.Cwe = ouv.Cwe + cod.VulType.En = ouv.EnDesc + cod.VulType.Zh = ouv.ZhDesc + } + var ouf models.OriginUpstreamFixSuggest + oufErr := models.GetOriginFixSuggest(ou.CveId, &ouf) + if oufErr == nil && ouf.FixId > 0 { + var oufr []models.OriginUpstreamFixSuggestRef + oufrNum, oufErr := models.GetOriginFixSuggestRef(ouf.FixId, &oufr) + if oufErr == nil && oufrNum > 0 { + for _, ref := range oufr { + var fr common.FixReferences + fr.Refsource = ref.Refsource + fr.Name = ref.Name + fr.Url = ref.Url + var oufrt []models.OriginUpstreamFixSuggestRefTag + oufrtNum, oufrtErr := models.GetOriginFixSuggestRefTag(ref.FixRefId, &oufrt) + if oufrtErr == nil && oufrtNum > 0 { + for _, tag := range oufrt { + fr.Tags = append(fr.Tags, tag.Name) + } + } + cod.FixSuggest.References = append(cod.FixSuggest.References, fr) + } + } + cod.FixSuggest.Detail = ouf.Detail + } + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + resp["body"] = cod + u.RetData(resp) + return + } else { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } +} diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go new file mode 100644 index 0000000000000000000000000000000000000000..0921b5e7c8d6235d08119dc2716ea53c6e35726e --- /dev/null +++ b/cve-vulner-manager/controllers/file.go @@ -0,0 +1,570 @@ +package controllers + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/taskhandler" + "cvevulner/util" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "math/rand" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +//FileController file operation routing processing +type FileController struct { + beego.Controller +} + +//DownloadLastExcel Download the latest excel file +// @router /lastExcel [get] +func (f *FileController) DownloadLastExcel() { + fd := beego.AppConfig.DefaultString("fileDir", "download") + err := common.CreateDir(fd) + er := models.ExportRecord{} + err = er.QueryLast() + if err != nil { + _ = f.Ctx.Output.Body([]byte("no file ")) + } + fp := filepath.Join(fd, er.FileName) + if ex, _ := util.IsExistPath(fp); !ex { + _ = f.Ctx.Output.Body([]byte("no file ")) + } + f.Ctx.Output.Download(fp, er.FileName) +} + +//DownLoadExcelByFileCode download excel file by code +//@router /downloadExcel +func (f *FileController) DownLoadExcelByFileCode() { + fd := beego.AppConfig.DefaultString("fileDir", "download") + common.CreateDir(fd) + fc := f.GetString("fileCode") + if fc == "" { + _ = f.Ctx.Output.Body([]byte("err: fileCode is a required parameter ")) + return + } + er := models.ExportRecord{FileCode: fc} + err := er.Read("file_code") + if err != nil { + _ = f.Ctx.Output.Body([]byte(fmt.Sprintf("err: %v", err))) + return + } + if er.FileName == "" { + _ = f.Ctx.Output.Body([]byte("err: Can not find excel file by fileCode! ")) + return + } + if er.State == 0 { + _ = f.Ctx.Output.Body([]byte("The file is being generated, please try again later!")) + return + } + if er.State == 2 { + _ = f.Ctx.Output.Body([]byte("File generation failed, please contact the administrator or regenerate!!")) + return + } + fp := filepath.Join(fd, er.FileName) + if ex, _ := util.IsExistPath(fp); !ex { + _ = f.Ctx.Output.Body([]byte("error:file does not exist")) + } + f.Ctx.Output.Download(fp, er.FileName) +} + +//TriggerCveData touch off generate cve data excel and get cve package +//@router /triggerCveData [get] +func (f *FileController) TriggerCveData() { + // Limit on the number of triggers + nameStr, limitCount := LimitTriggerSa() + if limitCount != 0 { + cha := fmt.Sprintf("The number of requests is too frequent, please try again later, " + + "there is currently a task being processed") + f.Ctx.WriteString(cha) + UpdateLimitTriggerSa(nameStr, 2) + return + } + startTime := f.GetString("startTime") + if startTime == "" { + f.Ctx.WriteString("Error: startTime cannot be empty") + UpdateLimitTriggerSa(nameStr, 2) + return + } + typeName := f.GetString("typeName") + if len(typeName) < 2 { + f.Ctx.WriteString("Error: typeName cannot be empty") + UpdateLimitTriggerSa(nameStr, 2) + return + } + legalBool := false + elt := models.EmailList{EmailType: 1} + el, eltErr := elt.Read("EmailType") + if eltErr != nil { + logs.Error("Failed to get mailing list, err: ", eltErr) + legalBool = true + } + for _, em := range el { + if strings.Contains(em.EmailName, "@") { + if strings.Split(em.EmailName, "@")[0] == typeName { + legalBool = true + } + } else { + legalBool = true + } + } + if !legalBool { + f.Ctx.WriteString("Error: The typeName parameter value is wrong") + UpdateLimitTriggerSa(nameStr, 2) + return + } + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("TriggerCveData, Issue token acquisition failed, "+ + "current time: ", common.GetCurTime()) + f.Ctx.WriteString("Error: Service internal error, try again later") + UpdateLimitTriggerSa(nameStr, 2) + return + } + owner := beego.AppConfig.String("gitee::owner") + // proc OpenEulerSaNum + taskhandler.UpdateUseOpenEulerSaNum() + rt := regexp.MustCompile(`^(\d{4})-\d{2}-(\d{2})$`) + find := rt.Match([]byte(startTime)) + if !find { + f.Ctx.WriteString(`Error: please enter the correct start time in a format like this "yyyy-MM-dd".`) + UpdateLimitTriggerSa(nameStr, 2) + return + } + unaffectYear, yerr := beego.AppConfig.Int("excel::unaffect_year") + if yerr != nil { + unaffectYear = 2018 + } + dir := beego.AppConfig.DefaultString("fileDir", "download") + common.CreateDir(dir) + var wgCrvf sync.WaitGroup + componentMap := make(map[string]taskhandler.ComponentInfo) + cvrfFileList := make(map[string][]string) + cvrfFileMap := make(map[string]taskhandler.CvrfSa) + affectBranchSlice := make([]string, 0) + var unaffectcvrf taskhandler.UnaffectCvrfSa + cvrffileName := filepath.Join(dir, "cvrf-unaffected-cve-"+common.GetCurDate()+".xml") + cvrfFileSlice, cvrfOk := cvrfFileList[taskhandler.UNAFFECTCVRFKEY] + if cvrfOk && len(cvrfFileSlice) > 0 { + cvrffileName = cvrfFileSlice[0] + } else { + cvrfNameSlice := make([]string, 0) + cvrfNameSlice = append(cvrfNameSlice, cvrffileName) + cvrfFileList[taskhandler.UNAFFECTCVRFKEY] = cvrfNameSlice + } + updateInfoSlice := make([]taskhandler.UpdateInfoXml, 0) + go f.Ctx.WriteString("Processing: Data is being prepared, please wait patiently, Please check your mail later!\n") + unaffectcvrf.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + unaffectcvrf.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + du := "openEuler-20.03-LTS@https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" + //du := beego.AppConfig.String("excel::v_pack_20_03_url") + du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) + csvPathList := strings.Split(du, ";") + if len(csvPathList) > 0 { + for _, csvP := range csvPathList { + openBranchx := strings.Split(csvP, "@") + if len(openBranchx) == 2 { + affectBranch := openBranchx[0] + taskhandler.UnaffectIssueProc(affectBranch, cvrfFileList, + componentMap, startTime, accessToken, owner, &unaffectcvrf, unaffectYear) + affectBranchSlice = append(affectBranchSlice, affectBranch) + csvDownPath := openBranchx[1] + now := time.Now().Unix() + en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", openBranchx[0], now) + fileCode := common.EncryptMd5(en) + wgCrvf.Add(1) + updateInfoxml := taskhandler.GenerateExcelTrigger(&wgCrvf, en, startTime, fileCode, affectBranch, + csvDownPath, dir, cvrfFileList, componentMap, cvrfFileMap) + updateInfoSlice = append(updateInfoSlice, updateInfoxml) + } + } + } + wgCrvf.Wait() + taskhandler.WriteUnaffectCvrfXml(cvrffileName, &unaffectcvrf) + taskhandler.SaveCvrfName(cvrfFileList, cvrffileName, taskhandler.UNAFFECTCVRFKEY) + // Generate cvrf file + GenAndUploadCvrf(cvrfFileMap, cvrfFileList, componentMap, affectBranchSlice, dir) + // Return the result first, continue processing the data + UpdateLimitTriggerSa(nameStr, 1) + GenUpdateInfoXmlFile(updateInfoSlice, dir) +} + +// Generate updateinfo xml file +func GenUpdateInfoXmlFile(updateInfoSlice []taskhandler.UpdateInfoXml, dir string) { + if len(updateInfoSlice) > 0 { + updateinfoDir := beego.AppConfig.String("obs::upload_updateinfo_dir") + obsDir := updateinfoDir + common.GetCurDate() + "/" + dirErr, objectDir := taskhandler.ObsCreateDir(obsDir) + for _, upInfo := range updateInfoSlice { + // write excel + if len(upInfo.XmfilePath) > 1 { + fileNameList := make([]string, 0) + fileExt := filepath.Ext(upInfo.XmfilePath) + excelName := upInfo.XmfilePath[:(len(upInfo.XmfilePath)-len(fileExt))] + ".xlsx" + taskhandler.CreateUpdateExcel(excelName) + zipFileList := []string{upInfo.FileName, upInfo.XmfilePath} + fileNameList = append(fileNameList, excelName) + zipFileList = append(zipFileList, excelName) + // write xml + taskhandler.WriteXml(upInfo.XmfilePath, excelName, upInfo.AffectBranch, upInfo.Cvexml, + upInfo.Dpdates, upInfo.SecurityNotice, upInfo.PackRpmx) + // zip file + if isExist := taskhandler.FileExist(zipFileList); isExist { + zipFileName := upInfo.AffectBranch + "_" + common.GetCurDate() + "_" + strconv.Itoa(time.Now().Nanosecond()) + ".zip" + zipFileName = filepath.Join(dir, zipFileName) + zipErr := taskhandler.ZipFiles(zipFileName, zipFileList, dir, dir) + if zipErr != nil { + logs.Error("File compression failed: err: ", zipErr) + } + // send email + sendError := taskhandler.SendEmail(zipFileName, 0, "", "") + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + return + } + zipFileList = append(zipFileList, zipFileName) + } + fileNameList = append(fileNameList, upInfo.XmfilePath) + // Upload to obs + if dirErr == nil && len(fileNameList) > 0 { + for _, fl := range fileNameList { + _, xmlFileName := filepath.Split(fl) + objectName := objectDir + xmlFileName + upErr := taskhandler.ObsUploadFile(objectName, fl) + if upErr != nil { + logs.Error("upErr: ", upErr) + } + } + } + // Clear file + taskhandler.DelFile(zipFileList) + } + } + } +} + +// Limit on the number of triggers +func LimitTriggerSa() (string, int) { + var captcha string + for i := 0; i < 6; i++ { + //产生0到9的整数 + num := rand.Intn(10) + //将整数转为字符串 + captcha += strconv.Itoa(num) + } + now := time.Now().UnixNano() + en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", captcha, now) + timeUnix := time.Now().Unix() + //It is time-consuming to generate excel, here is the current limit processing + er := models.ExportRecord{} + err := er.QueryLast() + if err == nil && (timeUnix-er.CreateTime < 300) { + logs.Error("The number of requests is too frequent, please try again later, " + + "there is currently a task being processed") + return "", 1 + } else { + rand.Seed(now) + fileCode := common.EncryptMd5(en) + er = models.ExportRecord{FileName: en, FileCode: fileCode, State: 0, CreateTime: timeUnix} + err = er.Insert() + logs.Error("err: ", err) + } + return en, 0 +} + +func UpdateLimitTriggerSa(fileName string, state int8) { + fr := models.ExportRecord{FileName: fileName} + err := fr.Read("file_name") + if err != nil { + logs.Error("fr.Read, err: ", err) + return + } + fr.State = state + _ = fr.Update("state") +} + +// generate and upload cvrf.xml +func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, + cvrfFileList map[string][]string, componentMap map[string]taskhandler.ComponentInfo, + affectBranchSlice []string, dir string) { + common.CreateDir(dir) + writeCvrfSlice := make([]string, 0) + totalFileSlice := make([]string, 0) + if len(componentMap) > 0 { + for comKey, comValue := range componentMap { + if len(affectBranchSlice) > 1 { + isEque := true + branchInfo1 := comValue.CveNumMap[affectBranchSlice[0]].CveNumSlice + for _, branchInfo := range affectBranchSlice[1:] { + if !common.CompareSlice(branchInfo1, comValue.CveNumMap[branchInfo].CveNumSlice) { + isEque = false + break + } + } + if isEque { + cvrfFileName := componentMap[comKey].CvrfFileName + cvrfVule := cvrfFileMap[cvrfFileName] + md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, + componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + if md5Ok { + UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) + var saf models.SaFileList + saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" + models.DeleteCvrfFileName(&saf, "FileName") + } + for _, br := range comValue.CveNumMap { + UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) + } + } else { + for _, br := range comValue.CveNumMap { + cvrfFileName := br.CvrfFileName + cvrfVule := cvrfFileMap[br.CvrfFileName] + md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(br.CveNumSlice, componentMap[comKey].OwnedComponent, + br.OpenEulerSANum, taskhandler.FIXEDFLAGE) + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + if md5Ok { + UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) + var saf models.SaFileList + saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" + models.DeleteCvrfFileName(&saf, "FileName") + } + } + UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) + } + } else { + cvrfFileName := componentMap[comKey].CvrfFileName + cvrfVule := cvrfFileMap[cvrfFileName] + md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, + componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + if md5Ok { + UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) + var saf models.SaFileList + saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" + models.DeleteCvrfFileName(&saf, "FileName") + } + for _, br := range comValue.CveNumMap { + UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) + } + } + } + } + cvrfFileList[taskhandler.CVRFFKEY] = writeCvrfSlice + uploadCvrfFile(cvrfFileList, totalFileSlice, dir, componentMap) +} + +func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, dir string, + componentMap map[string]taskhandler.ComponentInfo) { + uploadCvrfDir := beego.AppConfig.String("obs::upload_cvrf_dir") + downloadCvrfDir := beego.AppConfig.String("obs::download_cvrf_dir") + obsDir := uploadCvrfDir + common.GetCurDate() + "/" + dirErr, objectDir := taskhandler.ObsCreateDir(obsDir) + if dirErr != nil { + logs.Error("dirErr: ", dirErr) + } + indexFilePath := filepath.Join(dir, "index.txt") + indexObjectName := downloadCvrfDir + "index.txt" + downObsErr := taskhandler.ObsDownloadFile(indexObjectName, indexFilePath) + if downObsErr != nil { + logs.Error("downObsErr: ", downObsErr) + return + } + saFileStr := taskhandler.ReadFileAll(indexFilePath) + SaveFileToDb(saFileStr) + totalSlice := make([]string, 0) + saNumber := int64(1000) + sfl := models.GetCvrfAllFile() + curYears := strconv.Itoa(time.Now().Year()) + saDir := beego.AppConfig.DefaultString("saFileDir", "download/sa") + common.CreateAllDir(saDir) + if len(sfl) > 0 { + for _, l := range sfl { + fSlice := strings.Split(l.FileName, "-") + dirYears := curYears + if len(fSlice) > 4 { + dirYears = fSlice[3] + } + totalSlice = append(totalSlice, dirYears+"/"+l.FileName) + saNumber = l.SaNumber + } + } + // File name and data stored in database + fileSlice, fOk := cvrfFileList[taskhandler.CVRFFKEY] + if fOk { + totalFileSlice = UploadSaFile(fileSlice, totalSlice, totalFileSlice, saNumber, + curYears, saDir, dir, objectDir) + } + unaffFileSlice, unffOk := cvrfFileList[taskhandler.UNAFFECTCVRFKEY] + if unffOk { + subFileSlice := make([]string, 0) + updateFilePath := filepath.Join(dir, "update_unaffect.txt") + for _, fPath := range unaffFileSlice { + //taskhandler.WriteUnaffectCvrfXml(fPath, componentMap[fPath].UnaffectFile) + _, fileName := filepath.Split(fPath) + // File storage to db + recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, "", taskhandler.UNAFFECTFLAG) + logs.Info("recordErr: ", recordErr) + // Upload file, pending + subFileSlice = append(subFileSlice, fileName) + totalFileSlice = append(totalFileSlice, fPath) + // Upload successfully, modify file status + if len(fileName) > 5 { + openEulerSANum := fileName[5 : len(fileName)-4] + taskhandler.UpdateCvrfRecord(openEulerSANum, 2) + } + } + taskhandler.ReadWriteFile(updateFilePath, subFileSlice) + totalFileSlice = append(totalFileSlice, updateFilePath) + } + if len(totalFileSlice) > 0 { + for _, localFilePath := range totalFileSlice { + _, localFileName := filepath.Split(localFilePath) + obsFilePath := objectDir + localFileName + obsErr := taskhandler.PostFile(localFilePath, obsFilePath) + if obsErr != nil { + logs.Error("obsErr: ", obsErr) + } + } + dir := "download" + zipFileName := "cvrf-" + common.GetCurDate() + "_" + strconv.Itoa(time.Now().Nanosecond()) + ".zip" + zipFileName = filepath.Join(dir, zipFileName) + zipErr := taskhandler.ZipFiles(zipFileName, totalFileSlice, dir, dir) + if zipErr != nil { + logs.Error("File compression failed: err: ", zipErr) + } + // send email + sendError := taskhandler.SendEmail(zipFileName, 1, "", "") + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + return + } + totalFileSlice = append(totalFileSlice, zipFileName) + } + // Delete local files + taskhandler.DelFile(totalFileSlice) + logs.Info(cvrfFileList[taskhandler.BRANCHSKEY], ",End of generating cvrf format file this time") +} + +func UploadSaFile(fileSlice, totalSlice, totalFileSlice []string, saNumber int64, + curYears, saDir, dir, uploadPath string) []string { + subFileSlice := make([]string, 0) + oldFileSlice := make([]string, 0) + for _, fPath := range fileSlice { + _, fileName := filepath.Split(fPath) + oldSa := RegSa(fileName) + saNumber = saNumber + 1 + oldText := fmt.Sprintf("openEuler-SA-%v-%v", curYears, oldSa) + newText := fmt.Sprintf("openEuler-SA-%v-%v", curYears, saNumber) + newFile := fmt.Sprintf("cvrf-%v.xml", newText) + newPath := filepath.Join(saDir, newFile) + ReplaceFileSa(fPath, newPath, oldText, newText) + oldFileSlice = append(oldFileSlice, fPath) + // File storage to db + recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, newText, taskhandler.FIXEDFLAGE) + logs.Info("recordErr: ", recordErr) + // Upload file, pending + SaveFileRecord(newFile) + totalSlice = append(totalSlice, curYears+"/"+newFile) + taskhandler.ProcCvrfFileName(fileName) + subFileSlice = append(subFileSlice, curYears+"/"+newFile) + totalFileSlice = append(totalFileSlice, newPath) + // Upload successfully, modify file status + if len(newFile) > 5 { + openEulerSANum := newFile[5 : len(newFile)-4] + taskhandler.UpdateCvrfRecord(openEulerSANum, 2) + } + } + indexFilePath := filepath.Join(dir, "index.txt") + updateFilePath := filepath.Join(dir, "update_fixed.txt") + readErr := taskhandler.ReadWriteFile(indexFilePath, totalSlice) + if readErr != nil { + logs.Error(indexFilePath, readErr) + } + readErr = taskhandler.ReadWriteFile(updateFilePath, subFileSlice) + if readErr != nil { + logs.Error(updateFilePath, readErr) + } + totalFileSlice = append(totalFileSlice, indexFilePath) + totalFileSlice = append(totalFileSlice, updateFilePath) + taskhandler.DelFile(oldFileSlice) + return totalFileSlice +} + +func UpdateOpenEulerSaNumStatus(openEulerSANum string) { + var sa models.SaNumber + curYears := strconv.Itoa(time.Now().Year()) + sa.SaYears = curYears + sa.OpenEulerSANum = openEulerSANum + sa.UpdateTime = common.GetCurTime() + getErr := models.GetSaNumber(&sa, "OpenEulerSANum", "SaYears") + if getErr == nil { + sa.Status = 1 + models.UpdateSaNumber(&sa, "status") + } +} + +func RegSa(bufVule string) string { + subSlice1 := strings.Split(bufVule, ".") + if len(subSlice1) > 0 { + subSlice2 := strings.Split(subSlice1[0], "-") + if len(subSlice2) > 0 { + result := subSlice2[len(subSlice2)-1] + return result + } + } + return "" +} + +func SaveFileToDb(saFileStr string) { + if len(saFileStr) > 1 { + saFileSlice := strings.Split(saFileStr, "\n") + if len(saFileSlice) > 0 { + models.DeleteCvrfFileRecord() + for _, saf := range saFileSlice { + if len(saf) > 2 { + singSlice := strings.Split(saf, "/") + if len(singSlice) > 1 { + SaveFileRecord(singSlice[1]) + } + } + } + } + } +} + +func SaveFileRecord(fileName string) { + var af models.SaFileRecord + result2 := RegSa(fileName) + saNumber, _ := strconv.ParseInt(result2, 10, 64) + af.SaNumber = saNumber + af.Status = 1 + af.FileName = fileName + af.CreateTime = common.GetCurTime() + models.InsertCvrfFileRecord(&af) +} + +func ReplaceFileSa(oldPath, newPath, oldText, newText string) { + helper := taskhandler.ReplaceHelper{ + OldPath: oldPath, + NewPath: newPath, + OldText: oldText, + NewText: newText, + } + err := helper.DoWrok() + if err == nil { + fmt.Println("done!") + } else { + fmt.Println("error:", err.Error()) + } +} diff --git a/cve-vulner-manager/controllers/gauss.go b/cve-vulner-manager/controllers/gauss.go new file mode 100644 index 0000000000000000000000000000000000000000..83c4bbb303a4f54e7e9f216fdb60f94d5c38eb31 --- /dev/null +++ b/cve-vulner-manager/controllers/gauss.go @@ -0,0 +1,739 @@ +package controllers + +import ( + "cvevulner/common" + "cvevulner/errcode" + "cvevulner/models" + "cvevulner/taskhandler" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "os" + "regexp" + "sort" + "strings" +) + +// openGauss generates a summary of Sa data and provides an api interface for obtaining data +type GenSaController struct { + beego.Controller +} + +// Manually trigger, generate sa data, and then store the data in the database, a total of api calls +//@router / [get] +func (g *GenSaController) TriggerCveData() { + var gi taskhandler.GitInfo + startTime := g.GetString("startTime") + if startTime == "" { + g.Ctx.WriteString("Error: startTime cannot be empty") + return + } + gi.StartTime = startTime + releaseFlag, reErr := g.GetInt("releaseFlag", 1) + if reErr != nil { + g.Ctx.WriteString("Error: releaseFlag cannot be empty") + return + } + // Get configuration information + accessToken := os.Getenv("GITEE_GAUSS_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("TriggerCveData, Issue token acquisition failed, "+ + "current time: ", common.GetCurTime()) + g.Ctx.WriteString("Error: Failed to obtain gauss token") + return + } + gi.Token = accessToken + owner := beego.AppConfig.String("opengauss::gauss_owner") + gi.Owner = owner + // Check if the file is being processed + fileData, fileErr := models.QueryDownloadFile(2) + if len(fileData) > 0 { + logs.Info("There are currently files being processed, "+ + "please finish processing before triggering a new sa generation, file: ", fileData) + g.Ctx.WriteString("processing, There are currently files being processed, " + + "please finish processing before triggering a new sa generation") + return + } + // Query the file to be downloaded + fileData, fileErr = models.QueryDownloadFile(1) + if len(fileData) == 0 { + logs.Error("QueryDownloadFile, err: ", fileErr) + g.Ctx.WriteString("Error: There is currently no downloadable file path") + return + } + // Get file storage path + gaussDir := beego.AppConfig.DefaultString("gaussFileDir", "download/gauss") + common.CreateAllDir(gaussDir) + rt := regexp.MustCompile(`^(\d{4})-\d{2}-(\d{2})$`) + find := rt.Match([]byte(startTime)) + if !find { + g.Ctx.WriteString(`Error: please enter the correct start time in a format like this "yyyy-MM-dd".`) + return + } + // Define global variables to store cve information + cveComponent := taskhandler.CveComponent{} + branchVersion := make([]string, 0) + for _, fp := range fileData { + branchVersion = append(branchVersion, fp.RepoVersion) + } + // version, descending sort + sort.Sort(sort.Reverse(sort.StringSlice(branchVersion))) + // Clear unaffected historical data + taskhandler.ClearHisDataSa(1) + // Generate unaffected cve data + if len(branchVersion) > 0 { + taskhandler.GaussUnaffectIssue(branchVersion, gi) + } + // Clear historical data + taskhandler.ClearHisDataSa(2) + saInitValue := int64(0) + // Get sa initialization variable + ogs := models.OpenGaussSiteList{} + qgErr := models.QueryGaussSaMaxValue(&ogs) + if ogs.GaussId == 0 { + logs.Info("QueryGaussSaMaxValue, qgErr: ", qgErr) + saInitValue = beego.AppConfig.DefaultInt64("opengauss::sa_init_value", 1001) + } else { + saInitValue = ogs.SaNum + 1 + } + for _, fExcel := range fileData { + if releaseFlag == 1 { + g.Ctx.WriteString(fmt.Sprintf("start：The SA pre-release generation of openGauss has been triggered. "+ + "Please wait patiently for the completion of the data generation. "+ + "It is estimated that it will take a few minutes.", startTime, fExcel.FilePath)) + } else { + g.Ctx.WriteString(fmt.Sprintf("start：Currently, openGauss SA has been officially released and generated. "+ + "Please wait patiently for the completion of the data generation. "+ + "It is estimated that it will take a few minutes.", startTime, fExcel.FilePath)) + } + models.UpdateDownloadFile(fExcel.FileId, 2) + taskhandler.DownloadFileAndParse(fExcel, startTime, gaussDir, &cveComponent) + models.UpdateDownloadFile(fExcel.FileId, 1) + // Return the result first, continue processing the data + if releaseFlag == 1 { + g.Ctx.WriteString(fmt.Sprintf("Success：The SA pre-release generation of openGauss has been triggered. "+ + "Please wait patiently for the completion of the data generation. "+ + "It is estimated that it will take a few minutes.", startTime, fExcel.FilePath)) + } else { + g.Ctx.WriteString(fmt.Sprintf("Success：Currently, openGauss SA has been officially released and generated. "+ + "Please wait patiently for the completion of the data generation. "+ + "It is estimated that it will take a few minutes.", startTime, fExcel.FilePath)) + } + } + // Merging of multi-branch data + MergMultiBranchData(branchVersion, cveComponent, &saInitValue) + // Upgrade pre-release data to official data + if releaseFlag == 2 { + models.UpdateGaussPreTRelease() + } +} + +func MergMultiBranchData(branchVersion []string, cveComponent taskhandler.CveComponent, saInitValue *int64) { + bVersionSlice := make([]string, 0) + if cveComponent.PackMap != nil && len(cveComponent.PackMap) > 0 { + for k, v := range cveComponent.PackMap { + isEque := true + iFlag := 0 + for i, bv := range branchVersion { + _, ok1 := v.BranchMap[bv] + if ok1 { + iFlag = i + break + } + } + bv1 := v.BranchMap[branchVersion[iFlag]] + if (iFlag + 1) <= (len(branchVersion) - 1) { + for _, branchInfo := range branchVersion[iFlag+1:] { + bv2, ok2 := v.BranchMap[branchInfo] + if ok2 { + if !common.CompareSlice(bv1, bv2) { + isEque = false + break + } + } + + } + } + for _, bv := range branchVersion { + _, ok1 := v.BranchMap[bv] + if ok1 { + addBs := false + for _, vj := range bVersionSlice { + if vj == bv { + addBs = true + break + } + } + if !addBs { + bVersionSlice = append(bVersionSlice, bv) + } + } + } + taskhandler.PreGeneratedSaData(k, bVersionSlice, isEque, saInitValue) + } + } +} + +type GaussSaController struct { + beego.Controller +} + +func (c *GaussSaController) RetSaData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type GaussSaData struct { + SaId int64 `json:"saId"` + GaussSaNum string `json:"gaussSaNum"` + Summary string `json:"summary"` + CveLevel string `json:"cveLevel"` + CveLevelValue int `json:"cveLevelValue"` + AffectProduct string `json:"affectProduct"` + InfluenceComponent string `json:"influenceComponent"` + ReleaseDate string `json:"releaseDate"` +} + +// @Get the information of the SA that has been generated +// @Description get gauss sa +// @router / [get] +func (u *GaussSaController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var gs []GaussSaData + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + resp["body"] = []GaussSaData{} + resp["totalCount"] = 0 + resp["totalPage"] = 0 + //defer u.RetSaData(resp) + cveLevel, cErr := u.GetInt("cveLevel", 0) + if cErr != nil { + cveLevel = 0 + } + years, yErr := u.GetInt("years", 0) + if yErr != nil { + years = 0 + } + releaseFlag, rErr := u.GetInt("releaseFlag", 1) + if rErr != nil { + releaseFlag = 1 + } + searchName := u.GetString("searchName", "") + count := models.QueryGaussSaCount(cveLevel, years, releaseFlag, searchName) + if count > 0 { + resp["totalCount"] = count + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + pageNum, err := u.GetInt("pageNum", 1) + if err != nil { + logs.Error("err: ", err, ", pageNum: ", pageNum) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetSaData(resp) + return + } + pageSize, err := u.GetInt("pageSize", 100) + if err != nil { + logs.Error("err: ", err, ", pageSize: ", pageSize) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetSaData(resp) + return + } + if int(count) % pageSize == 0 { + resp["totalPage"] = int(count) / pageSize + } else { + resp["totalPage"] = int(count) / pageSize + 1 + } + saData, saErr := models.QueryOpenGaussSiteList(pageNum, pageSize, cveLevel, years, releaseFlag, searchName) + if len(saData) > 0 { + for _, sa := range saData { + var irda GaussSaData + irda.SaId = sa.GaussId + irda.GaussSaNum = sa.GaussSaNum + irda.Summary = sa.Summary + irda.CveLevel = sa.CveLevel + irda.CveLevelValue = sa.CveLevelValue + irda.InfluenceComponent = sa.InfluenceComponent + if len(sa.CreateTime) > 10 { + irda.ReleaseDate = sa.CreateTime[:10] + } else { + irda.ReleaseDate = sa.CreateTime + } + irda.AffectProduct = sa.AffectProduct + irda.AffectProduct = strings.ReplaceAll(irda.AffectProduct, "/", " / ") + gs = append(gs, irda) + } + resp["body"] = gs + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetSaData(resp) + } else { + logs.Error("QueryOpenGaussSiteList, saErr: ", saErr) + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetSaData(resp) + return + } + } +} + +type GaussTag struct { + PackageName string `json:"packageName"` + AffectedPlatform []string `json:"affectedPlatform"` +} + +type GaussPackage struct { + GroupName string `json:"groupName"` + GaussTag []GaussTag `json:"tagBody"` +} + +type GaussVersions struct { + Versions string `json:"versions"` + GaussPackage []GaussPackage `json:"packageBody"` +} + +type GaussSaDetailData struct { + GaussSaNum string `json:"gaussSaNum"` + Summary string `json:"summary"` + CveLevel string `json:"cveLevel"` + CveLevelValue int `json:"cveLevelValue"` + AffectProduct string `json:"affectProduct"` + InfluenceComponent string `json:"influenceComponent"` + ReleaseDate string `json:"releaseDate"` + Introduction string `json:"introduction"` + Theme string `json:"theme"` + Description string `json:"description"` + CveNumbers string `json:"cveNumbers"` + ReferenceLink string `json:"referenceLink"` + GaussVersions []GaussVersions `json:"versionsBody"` +} + +type GaussSaDetailController struct { + beego.Controller +} + +func (c *GaussSaDetailController) RetSaDetailData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title Get GaussSaDetailController +// @Description get sa detail +// @router / [get] +func (u *GaussSaDetailController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var ird GaussSaDetailData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = GaussSaDetailData{} + //defer u.RetSaDetailData(resp) + gaussSaNum := u.GetString("gaussSaNum", "") + if gaussSaNum == "" { + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetSaDetailData(resp) + return + } + osl := models.OpenGaussSiteList{GaussSaNum: gaussSaNum} + oslErr := models.QueryOpenGaussSaByNum(&osl, "GaussSaNum") + if osl.GaussId > 0 { + CreateSaDetailData(&ird, osl) + resp["body"] = ird + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetSaDetailData(resp) + } else { + logs.Error("QueryOpenGaussSaByNum, oslErr: ", oslErr) + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetSaDetailData(resp) + return + } +} + +func CreateSaDetailData(ird *GaussSaDetailData, osl models.OpenGaussSiteList) { + nvdRelink := beego.AppConfig.String("opengauss::nvd_relink") + ird.GaussSaNum = osl.GaussSaNum + ird.AffectProduct = osl.AffectProduct + ird.Summary = osl.Summary + ird.Introduction = osl.Introduction + ird.CveLevel = osl.CveLevel + ird.CveLevelValue = osl.CveLevelValue + ird.InfluenceComponent = osl.InfluenceComponent + if len(osl.CreateTime) > 10 { + ird.ReleaseDate = osl.CreateTime[:10] + } else { + ird.ReleaseDate = osl.CreateTime + } + ird.Theme = osl.Theme + ird.Description = osl.Description + if strings.Index(ird.Description, "\n\n") >= 0 { + ird.Description = strings.ReplaceAll(ird.Description, "\n\n", "
") + }else if strings.Index(ird.Description, "\n") >= 0 { + ird.Description = strings.ReplaceAll(ird.Description, "\n", "
") + } + ird.CveNumbers = osl.CveNums + cveLinkSlice := make([]string, 0) + if len(osl.CveNums) > 1 { + cveNumSlice := strings.Split(osl.CveNums, ";") + if len(cveNumSlice) > 0 { + for _, cve := range cveNumSlice { + cveLinkSlice = append(cveLinkSlice, nvdRelink+cve) + } + } + } + if len(cveLinkSlice) > 0 { + ird.ReferenceLink = strings.Join(cveLinkSlice, ";") + } + gv := make([]GaussVersions, 0) + // Query the associated version information + if len(osl.AffectProduct) > 0 { + affectSlice := strings.Split(osl.AffectProduct, "/") + if len(affectSlice) > 0 { + for _, affect := range affectSlice { + var lgv GaussVersions + lgv.Versions = affect + ogp, pErr := models.QueryGaussPackageByGid(osl.GaussId, affect) + if len(ogp) > 0 { + groupMap := make(map[int64]map[string][]int64) + for _, gp := range ogp { + groupV, ok := groupMap[gp.GroupId] + if !ok { + platMap := make(map[string][]int64) + paltSlice := make([]int64, 0) + paltSlice = append(paltSlice, gp.PlatId) + platMap[gp.RpmName] = paltSlice + groupMap[gp.GroupId] = platMap + } else { + platV, ok := groupV[gp.RpmName] + if !ok { + paltSlice := make([]int64, 0) + paltSlice = append(paltSlice, gp.PlatId) + groupV[gp.RpmName] = paltSlice + } else { + vIsExit := false + for _, pv := range platV { + if pv == gp.PlatId { + vIsExit = true + break + } + } + if !vIsExit { + platV = append(platV, gp.PlatId) + groupV[gp.RpmName] = platV + } + } + } + } + if len(groupMap) > 0 { + gp := make([]GaussPackage, 0) + for kg, vg := range groupMap { + ogg := models.OpenGaussGroup{GroupId: kg} + gErr := models.QueryOpenGaussGroup(&ogg, "GroupId") + if gErr == nil { + var ge GaussPackage + ge.GroupName = ogg.GroupName + tag := make([]GaussTag, 0) + for pgk, pgv := range vg { + var tagv GaussTag + tagv.PackageName = pgk + ogl, opErr := models.QueryGaussPlatByPids(pgv) + if len(ogl) > 0 { + plSlice := make([]string, 0) + for _, pl := range ogl { + plSlice = append(plSlice, pl.PlatName) + } + tagv.AffectedPlatform = plSlice + } else { + logs.Error("QueryGaussPlatByPids, opErr: ", opErr) + } + tag = append(tag, tagv) + } + ge.GaussTag = tag + gp = append(gp, ge) + } + } + lgv.GaussPackage = gp + } + } else { + logs.Error("QueryGaussPackageByGid, pErr: ", pErr) + } + gv = append(gv, lgv) + } + } + } + ird.GaussVersions = gv +} + +type GaussCveController struct { + beego.Controller +} + +func (c *GaussCveController) RetGaussCveData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type GaussCveData struct { + CveId int64 `json:"cveId"` + CveNum string `json:"cveNum"` + Description string `json:"description"` + NvdScore float64 `json:"NVDScore"` + UpdateTime string `json:"updateTime"` + ReleaseDate string `json:"releaseDate"` +} + +// @Get the cve information of gauss +// @Description get gauss cve +// @router / [get] +func (u *GaussCveController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var gs []GaussCveData + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + resp["body"] = []GaussCveData{} + resp["totalCount"] = 0 + resp["totalPage"] = 0 + //defer u.RetGaussCveData(resp) + releaseFlag, rErr := u.GetInt("releaseFlag", 1) + if rErr != nil { + releaseFlag = 1 + } + isAffectFlag, rErr := u.GetInt("isAffectFlag", 1) + if rErr != nil { + isAffectFlag = 1 + } + searchName := u.GetString("searchName", "") + count := models.QueryGaussCveCount(releaseFlag, isAffectFlag, searchName) + if count > 0 { + resp["totalCount"] = count + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + pageNum, err := u.GetInt("pageNum", 1) + if err != nil { + logs.Error("err: ", err, ", pageNum: ", pageNum) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetGaussCveData(resp) + return + } + pageSize, err := u.GetInt("pageSize", 100) + if err != nil { + logs.Error("err: ", err, ", pageSize: ", pageSize) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetGaussCveData(resp) + return + } + if int(count) % pageSize == 0 { + resp["totalPage"] = int(count) / pageSize + } else { + resp["totalPage"] = int(count) / pageSize + 1 + } + saData, saErr := models.QueryOpenGaussCveList(pageNum, pageSize, releaseFlag, isAffectFlag, searchName) + if len(saData) > 0 { + for _, sa := range saData { + var irda GaussCveData + irda.CveId = sa.Id + irda.CveNum = sa.CveNum + irda.Description = sa.Description + irda.NvdScore = sa.NVDScore + irda.UpdateTime = sa.UpdateTime + if len(sa.CreateTime) > 10 { + irda.ReleaseDate = sa.CreateTime[:10] + } else { + irda.ReleaseDate = sa.CreateTime + } + gs = append(gs, irda) + } + resp["body"] = gs + resp["code"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetGaussCveData(resp) + } else { + logs.Error("QueryOpenGaussCveList, saErr: ", saErr) + resp["code"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetGaussCveData(resp) + return + } + } +} + +type GaussCveAffectProduct struct { + AffectProduct string `json:"affectProduct"` + PackName string `json:"packName"` + FixLabel string `json:"fixLabel"` +} + +type GaussCveSa struct { + GaussSaNum string `json:"gaussSaNum"` + SaId int64 `json:"saId"` + Summary string `json:"summary"` + ReleaseDate string `json:"releaseDate"` +} + +type GaussCveCvssV3 struct { + NvdScore float64 `json:"NVDScore"` + OpenGaussScore float64 `json:"openGaussScore"` + NAttackVector string `json:"nAttackVector"` + OAttackVector string `json:"oAttackVector"` + NAttackComplexity string `json:"nAttackComplexity"` + OAttackComplexity string `json:"oAttackComplexity"` + NPrivilegeRequired string `json:"nPrivilegeRequired"` + OPrivilegeRequired string `json:"oPrivilegeRequired"` + NUserInteraction string `json:"nUserInteraction"` + OUserInteraction string `json:"oUserInteraction"` + NScope string `json:"nScope"` + OScope string `json:"oScope"` + NConfidentiality string `json:"nConfidentiality"` + OConfidentiality string `json:"oConfidentiality"` + NIntegrity string `json:"nIntegrity"` + OIntegrity string `json:"oIntegrity"` + NAvailability string `json:"nAvailability"` + OAvailability string `json:"oAvailability"` + ScoreType string `json:"scoreType"` +} + +type GaussCveDetailData struct { + CveId int64 `json:"cveId"` + CveNum string `json:"cveNum"` + Description string `json:"description"` + NvdScore float64 `json:"NVDScore"` + UpdateTime string `json:"updateTime"` + ReleaseDate string `json:"releaseDate"` + CVSSV3 GaussCveCvssV3 `json:"CVSSV3"` + SaBody GaussCveSa `json:"saBody"` + AffectBody []GaussCveAffectProduct `json:"affectBody"` +} + +type GaussCveDetailController struct { + beego.Controller +} + +func (c *GaussCveDetailController) RetCveDetailData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title Get GaussSaDetailController +// @Description get sa detail +// @router / [get] +func (u *GaussCveDetailController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var ird GaussCveDetailData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = GaussCveDetailData{} + //defer u.RetCveDetailData(resp) + cveNum := u.GetString("cveNum", "") + if cveNum == "" { + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetCveDetailData(resp) + return + } + osl := models.OpenGaussCveList{CveNum: cveNum} + oslErr := models.QueryOpenGaussCveByNum(&osl, "CveNum") + if osl.Id > 0 { + CreateCveDetailData(&ird, osl) + resp["body"] = ird + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetCveDetailData(resp) + } else { + logs.Error("QueryOpenGaussCveByNum, oslErr: ", oslErr) + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetCveDetailData(resp) + return + } +} + +func CreateCveDetailData(gdd *GaussCveDetailData, ocl models.OpenGaussCveList) { + gdd.CveId = ocl.Id + gdd.CveNum = ocl.CveNum + gdd.Description = ocl.Description + if strings.Index(gdd.Description, "\n\n") >= 0 { + gdd.Description = strings.ReplaceAll(gdd.Description, "\n\n", "
") + }else if strings.Index(gdd.Description, "\n") >= 0 { + gdd.Description = strings.ReplaceAll(gdd.Description, "\n", "
") + } + gdd.NvdScore = ocl.NVDScore + gdd.UpdateTime = ocl.UpdateTime + if len(ocl.CreateTime) > 10 { + gdd.ReleaseDate = ocl.CreateTime[:10] + } else { + gdd.ReleaseDate = ocl.CreateTime + } + var gcv GaussCveCvssV3 + gcv.NvdScore = ocl.NVDScore + gcv.OpenGaussScore = ocl.OpenEulerScore + gcv.NAttackVector = ocl.NattackVector + gcv.OAttackVector = ocl.OattackVector + gcv.NAttackComplexity = ocl.NattackComplexity + gcv.OAttackComplexity = ocl.OattackComplexity + gcv.NPrivilegeRequired = ocl.NprivilegeRequired + gcv.OPrivilegeRequired = ocl.OprivilegeRequired + gcv.NUserInteraction = ocl.NuserInteraction + gcv.OUserInteraction = ocl.OuserInteraction + gcv.NScope = ocl.Nscope + gcv.OScope = ocl.Oscope + gcv.NConfidentiality = ocl.Nconfidentiality + gcv.OConfidentiality = ocl.Oconfidentiality + gcv.NIntegrity = ocl.Nintegrity + gcv.OIntegrity = ocl.Ointegrity + gcv.NAvailability = ocl.Navailability + gcv.OAvailability = ocl.Oavailability + gcv.ScoreType = ocl.ScoreType + gdd.CVSSV3 = gcv + if ocl.GaussId > 0 { + ost := models.OpenGaussSiteList{GaussId: ocl.GaussId} + ostErr := models.QueryOpenGaussSaByNum(&ost, "GaussId") + if ostErr == nil { + var gs GaussCveSa + gs.SaId = ost.GaussId + gs.Summary = ost.Summary + gs.GaussSaNum = ost.GaussSaNum + if len(ost.CreateTime) > 10 { + gs.ReleaseDate = ost.CreateTime[:10] + } else { + gs.ReleaseDate = ost.CreateTime + } + gdd.SaBody = gs + } + } else { + gdd.SaBody = GaussCveSa{} + } + ogv, ogvErr := models.QueryGaussVersionByCid(gdd.CveId) + if len(ogv) > 0 { + gcp := make([]GaussCveAffectProduct, 0) + for _, gv := range ogv { + var gp GaussCveAffectProduct + gp.AffectProduct = gv.RepoVersion + gp.PackName = ocl.PackName + gp.FixLabel = gv.AffectStatusName + gcp = append(gcp, gp) + } + gdd.AffectBody = gcp + } else { + logs.Error("QueryGaussVersionByCid, ogvErr: ", ogvErr) + } +} diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go new file mode 100644 index 0000000000000000000000000000000000000000..01e1a912f6b70ea3275fe0de1f79c99b71b88364 --- /dev/null +++ b/cve-vulner-manager/controllers/hook.go @@ -0,0 +1,3375 @@ +package controllers + +import ( + "bytes" + "cvevulner/common" + "cvevulner/cve-timed-task/tabletask" + "cvevulner/models" + "cvevulner/task" + "cvevulner/taskhandler" + "cvevulner/util" + "encoding/json" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io/ioutil" + "net/http" + "os" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +var ( + //GiteeUserAgent gitee hook request flag + GiteeUserAgent = "git-oschina-hook" + //XGiteeToken password or sign + XGiteeToken = "X-Gitee-Token" + //XGIteeEventType webhook event type + XGIteeEventType = "X-Gitee-Event" + //NoteHookType type of comment + NoteHookType = "Note Hook" + //PullReqHookType type of pull request + PullReqHookType = "Merge Request Hook" + //PushTagHookType type of push or tag + PushTagHookType = "Tag Push Hook" + //IssueHookType type of issue + IssueHookType = "Issue Hook" +) + +const ( + //CommentAnalysisCplTpl complete comment analysis reply template + CommentAnalysisCplTpl = "@%v %v" + //ReviewPrivateLettersTpl send private review letters template + ReviewPrivateLettersTpl = `%s(%s)analysis is over,CVEScore:%v;OpenEulerScore:%v.Please review!` + //ReviewRejectScore reply the review reject template + ReviewRejectScore = `@%v you submit issue score audit failed(reject by %v),Please re-analyze and submit!` + //ReviewApproveScore replay the review approve template + ReviewApproveScore = `@%v you submit issue score audit success(approved by %v),You can proceed to the next step!` + //CommentReviewTpl comment review template + CommentReviewTpl = `%v The CVE score needs to be reviewed (the review instruction /approve or /reject means agreement and rejection).` + //IssueRejectState issue state rejected + IssueRejectState = "rejected" + //IssueCloseState issue state closed + IssueCloseState = "closed" + //IssueProgressState issue state progressing + IssueProgressState = "progressing" + //IssueOpenState issue state open + IssueOpenState = "open" + //AnalysisComplete issue analysis complete comment + AnalysisComplete = "@%v 经过 cve-manager 解析, 已分析的内容如下表所示:\n" + // Content review tips + ContentReview = "%v 请完成以下操作:\n" + // Not filling in the correct format + IssueErroFormat = "%v 经过 cve-manager 解析, 填写openEuler评分未通过安全组成员审核需要再次在评论区提交评分,通过审核后才能关闭ISSUE." + // Remind the security group to review + CommentReviewRemind = "%v 经过 cve-manager 解析 openEuler评分 已改变 需要您及时进行审核,以便maintainer进行后续操作." + // Review private messages + CommentPrivateReview = "%v 仓库的CVE和安全问题的ISSUE,需要您进行审核,CVE编号: %v" + // Rating review failed + CommentPrivateOpenEuler = "%v 仓库的CVE和安全问题的ISSUE, CVE编号: %v, 填写openEuler评分未通过安全组成员审核需要再次在评论区提交评分,通过审核后才能关闭ISSUE." + // Review reminder + CommentReviewRemindMaintainer = "@%v 经过 cve-manager 解析 openEuler评分 已改变 需要等待安全组成员审核通过以后, 才能进行后续操作." + CIssueType = "CVE和安全问题" + HasCreateIssue = "@%v 当前CVE:%v 已经创建过对应的ISSUE, 请不要重复创建ISSUE, cve-manager 只会识别之前的ISSUE,不会对当前ISSUE进行处理." + CommentCheckVersion = "@%v 请确认分支: %v 受影响/不受影响." + // rejected or upend + CommentRejectedState = `@%v 当前issue状态为: %v,请先修改issue状态, 否则评论无法被识别.` + // Get cve information comment + CommentGetNvdCveSuccess = `@%v CVE信息从NVD同步成功, 稍后请重新加载页面.` + CommentGetNvdCveFailed = `@%v CVE信息从NVD同步失败, 请稍后重试, 或者数据源不存在.` + CommentRepeatIssue = `%v 请检查当前: %v,是否重复创建, issue编号： %v, 重复创建的issue,将不会被再次识别.` +) + +var comLock sync.Mutex + +//HookEventControllers gitee hook callback +type HookEventControllers struct { + beego.Controller +} + +type AgencyPrams struct { + CveUrl string + PatchUrl string +} + +//Post handle gitee webhook +// @router / [post] +func (c *HookEventControllers) Post() { + if ok := c.isLegitimateHookEvent(); !ok { + c.Ctx.ResponseWriter.WriteHeader(406) + c.Ctx.WriteString("Illegal incident, discarded") + return + } + eventType := c.Ctx.Request.Header.Get(XGIteeEventType) + c.Ctx.ResponseWriter.WriteHeader(200) + c.Ctx.WriteString("Event received: " + eventType) + switch eventType { + case NoteHookType: //handle comment hook data + c.handleNoteDate() + case PullReqHookType: + c.handlePullReq() + case IssueHookType: + c.handleIssue() + case PushTagHookType: + c.handlePushTag() + default: + logs.Info(eventType) + } +} + +//isLegitimateHookEvent according to gitee doc judge +func (c *HookEventControllers) isLegitimateHookEvent() (ok bool) { + ok = true + //judge user agent + uAgent := c.Ctx.Request.Header.Get("User-Agent") + if uAgent != GiteeUserAgent { + ok = false + } + ctType := c.Ctx.Request.Header.Get("Content-Type") + if "application/json" != ctType { + ok = false + } + //judge hook password + xToken := c.Ctx.Request.Header.Get(XGiteeToken) + //logs.Info(xToken) + hookPwd := beego.AppConfig.String("hook::hookpwd") + if xToken != hookPwd { + logs.Error("hookPwd Err, xToken: ", xToken) + } + return +} + +func (c *HookEventControllers) handleNoteDate() { + var hookNote models.CommentPayload + err := json.Unmarshal(c.Ctx.Input.RequestBody, &hookNote) + if err != nil { + logs.Error(err, "\n, ", string(c.Ctx.Input.RequestBody)) + return + } + hookPwd := beego.AppConfig.String("hook::hookpwd") + hookNote.Password = util.TrimString(hookNote.Password) + hookPwd = util.TrimString(hookPwd) + if hookNote.Action == "comment" && hookNote.NoteableType == "Issue" && hookNote.Password == hookPwd { + logs.Info(string(c.Ctx.Input.RequestBody)) + //handle issue comment + go handleIssueComment(hookNote) + } +} + +func (c *HookEventControllers) handlePullReq() { + +} + +func (c *HookEventControllers) handlePushTag() { + +} + +func (c *HookEventControllers) handleIssue() { + logs.Info(string(c.Ctx.Input.RequestBody)) + issueHook := models.IssuePayload{} + err := json.Unmarshal(c.Ctx.Input.RequestBody, &issueHook) + if err != nil { + logs.Error(err, "\n, RequestBody: ", string(c.Ctx.Input.RequestBody)) + return + } + cuAccount := issueHook.Sender.Login + if issueHook.Issue.Number == "" || cuAccount == "" { + logs.Error("Data has null values: issueNum, cuAccount: ", issueHook.Issue.Number, cuAccount) + return + } + nameSpace := util.TrimString(issueHook.Repository.NameSpace) + organizationID := int8(1) + organizationID = taskhandler.GetOrganizationId(nameSpace) + botCuAccountStr := beego.AppConfig.String("cve::bot_cu_account") + botCuAccountList := strings.Split(botCuAccountStr, ",") + if len(botCuAccountList) > 0 { + for _, botCu := range botCuAccountList { + if cuAccount == botCu { + logs.Error(cuAccount, ", Ignore this comment") + return + } + } + } + hookPwd := beego.AppConfig.String("hook::hookpwd") + issueHook.Password = util.TrimString(issueHook.Password) + if issueHook.Password != hookPwd { + logs.Error("Hook callback pwd verification error, hook: ", issueHook) + return + } + if issueHook.Action == "assign" { + //Update the person in charge of the issue template + issueTmp := models.IssueTemplate{IssueNum: issueHook.Iid, IssueId: issueHook.Issue.Id} + err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if err != nil { + logs.Error(err, ",issueTmp: ", issueTmp) + return + } + issueTmp.Assignee = issueHook.Assignee.Login + err = models.UpdateIssueTemplate(&issueTmp, "issue_assignee") + if err != nil { + logs.Error(err, ",issueTmp: ", issueTmp) + } + } + if issueHook.Action == "state_change" { + //handle issue state change + err = handleIssueStateChange(&issueHook) + if err != nil { + logs.Error(err, "\n,", issueHook) + return + } + } + if issueHook.Action == "open" { + owner, token := common.GetOwnerAndToken(organizationID) + issueTmp := models.IssueTemplate{IssueNum: issueHook.Iid, IssueId: issueHook.Issue.Id} + err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if err == nil && issueTmp.TemplateId > 0 { + vc := models.VulnCenter{CveId: issueTmp.CveId} + vcErr := models.GetVulnCenterByCid(&vc, "CveId") + if vcErr == nil && vc.CveId > 0 && vc.OrganizationID == organizationID { + //cc := fmt.Sprintf(CommentRepeatIssue, "@"+cuAccount, issueTmp.CveNum, issueTmp.IssueNum) + //taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + logs.Error("Duplicate webhook data is discarded directly, cveNum:", vc.CveNum, ",packName: ", vc.RepoName) + return + } + } + err = gitAddIssueProc(&issueHook, organizationID) + if err != nil { + logs.Error(err) + } + issueTmp = models.IssueTemplate{IssueNum: issueHook.Iid, IssueId: issueHook.Issue.Id} + err = models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if err == nil { + PostTriggerGetCve(issueTmp, owner, token, cuAccount) + } + return + } + if issueHook.Action == "delete" { + err = gitDelIssueProc(&issueHook, organizationID) + if err != nil { + logs.Error(err, "\n, ", issueHook) + return + } + } +} + +func closeIssuePrivilage(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, + token, owner, fixed, unFix string, cveCenter *models.VulnCenter) bool { + closePrBool := true + if issueHook.Sender.UserName != "" && len(issueHook.Sender.UserName) > 1 { + if isReviewer(util.TrimString(issueHook.Sender.UserName)) { + if msg, tb, ok := checkIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + issueTmp.IssueStatus = 1 + issueTmp.MtAuditFlag = 1 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, "@"+issueHook.Sender.UserName) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v,", issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(token, content+msg, issueHook.Issue.Assignee.Login) + } + } else { + issueTmp.StatusName = issueHook.Issue.StateName + issueTmp.SaAuditFlag = 1 + issueTmp.MtAuditFlag = 1 + issueTmp.OpAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + issueTmp.IssueLabel = fixed + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = unFix + } + } + closePrBool = false + } + } + return closePrBool +} + +func sigReviewSend(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, + token, owner, fixed, unFix, assignee string, cveCenter *models.VulnCenter) { + issueTmp.IssueStatus = 1 + issueTmp.Status = 1 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + *cveCenter, *issueTmp) + if issueErr == nil { + if issueTmp.OpAuditFlag == 2 { + cc := fmt.Sprintf(IssueErroFormat, assignee) + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + content := fmt.Sprintf(CommentPrivateOpenEuler, issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(token, content, issueHook.Sender.UserName) + } else if issueTmp.OpAuditFlag == 0 { + list, revErr := models.GetSecurityReviewerList() + if revErr == nil && len(list) > 0 { + content := fmt.Sprintf(CommentPrivateReview, issueTmp.Repo, issueTmp.CveNum) + ns := make([]string, len(list)) + for k, v := range list { + ns[k] = "@" + v.NameSpace + " " + taskhandler.SendPrivateLetters(token, content, v.NameSpace) + } + if len(ns) > 0 { + cc := fmt.Sprintf(CommentReviewRemind, strings.Join(ns, ",")) + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + } + } else { + logs.Error("revErr: ", revErr) + } + } + } +} + +func getMaintainer(path, prSender, assignee string) string { + maintainerList, mainOk := models.QueryRepoAllMaintainer(path) + assList := []string{} + if mainOk && len(maintainerList) > 0 { + for _, v := range maintainerList { + assList = append(assList, "@"+v.MemberName+" ") + } + } + if prSender != "" && len(prSender) > 1 { + isListBool := common.IsValueInList("@"+prSender+" ", assList) + if !isListBool { + assList = append(assList, "@"+prSender+" ") + } + } + maintainerVaule := "" + if len(assList) > 0 { + maintainerVaule = strings.Join(assList, ",") + } else { + maintainerVaule = "@" + assignee + " " + } + return maintainerVaule +} + +func otherCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, + token, owner, fixed, unFix, path string, cveCenter *models.VulnCenter) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, + issueTmp.OwnedVersion, cveCenter.OrganizationID) + if len(unFixList) > 0 { + //send comment to issue + issueTmp.IssueStatus = 1 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, issueHook.Sender.UserName, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v,", issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(token, content, issueHook.Sender.UserName) + } + } else { + commonFunc := checkGaussIssueClosedAnalysisComplete + if cveCenter.OrganizationID == 4 { + commonFunc = checkLooKengIssueClosedAnalysisComplete + } else if cveCenter.OrganizationID == 3 { + commonFunc = checkSporeIssueClosedAnalysisComplete + } + if _, tb, ok := commonFunc(issueTmp); !ok { + //send comment to issue + issueTmp.IssueStatus = 1 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, path, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, "@"+issueHook.Sender.UserName) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + } + } else { + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + issueTmp.Status = 1 + assignee := "@" + issueTmp.Assignee + issuePrFlag := VerifyIssueAsPr(issueTmp, *cveCenter, false, + assignee, issueHook.Sender.UserName) + if issuePrFlag { + //1. change issue status + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + issueTmp.StatusName = issueHook.Issue.StateName + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + issueTmp.IssueLabel = fixed + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = unFix + } + } else { + issueTmp.IssueStatus = 1 + cveCenter.IsExport = 0 + } + } + } +} + +func closeIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, + token, owner, fixed, unFix string, cveCenter *models.VulnCenter) { + closePrBool := true + closeIssuePrFlag, closeOk := beego.AppConfig.Int64("cve::close_issue_privilege") + if closeOk == nil && closeIssuePrFlag == 1 { + closePrBool = closeIssuePrivilage(issueHook, issueTmp, + token, owner, fixed, unFix, cveCenter) + } + if closePrBool { + issueTmp.Status = 1 + cveCenter.IsExport = 0 + issueTmp.MtAuditFlag = 1 + assignee := getMaintainer(issueTmp.Repo, issueHook.Sender.UserName, issueTmp.Assignee) + openScoreFlag := true + if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.NVDScore > 0 && + issueTmp.OpAuditFlag != 1 && issueTmp.OpenEulerScore > 0 { + //send comment to issue + openScoreFlag = false + sigReviewSend(issueHook, issueTmp, token, owner, fixed, unFix, assignee, cveCenter) + } + if openScoreFlag { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, + issueTmp.OwnedVersion, cveCenter.OrganizationID) + if len(unFixList) > 0 { + //send comment to issue + issueTmp.IssueStatus = 1 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, issueHook.Sender.UserName, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v,", issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(token, content, issueHook.Sender.UserName) + } + } else { + if msg, tb, ok := checkIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + issueTmp.IssueStatus = 1 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, "@"+issueHook.Sender.UserName) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v,", issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(token, content+msg, issueHook.Sender.UserName) + } + } else { + //1. change issue status + issueTmp.IssueStatus = 2 + //issueTmp.Status = 3 + cveCenter.IsExport = 3 + if issueTmp.MtAuditFlag == 0 { + issueTmp.IssueStatus = 1 + issueTmp.Status = 1 + cveCenter.IsExport = 0 + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**issue关闭前,请确认模板分析内容的准确性与完整性,确认无误后,请在评论区输入: /approve, 否则无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + } + return + } + issueTmp.IssueLabel = unFix + issueTmp.StatusName = "open" + issueTmp.Status = 1 + issuePrFlag := VerifyIssueAsPr(issueTmp, *cveCenter, false, + assignee, issueHook.Sender.UserName) + if issuePrFlag { + issueTmp.StatusName = issueHook.Issue.StateName + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + issueTmp.IssueLabel = fixed + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = unFix + } + } else { + issueTmp.IssueStatus = 1 + cveCenter.IsExport = 0 + } + } + } + } + } +} + +// Entry function for handling issue status +func handleIssueStateChange(issueHook *models.IssuePayload) error { + unFix := beego.AppConfig.String("labelUnFix") + fixed := beego.AppConfig.String("labelFixed") + uNaffected := beego.AppConfig.String("labeUnaffected") + issueId := issueHook.Issue.Id + issueTmp := models.IssueTemplate{} + issueTmp.IssueId = issueId + issueTmp.IssueNum = issueHook.Iid + repoPath := "" + if issueHook.Issue.Repository.Path != "" && + len(issueHook.Issue.Repository.Path) > 1 { + repoPath = issueHook.Issue.Repository.Path + } + issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if issueErr != nil { + return issueErr + } + path := issueTmp.Repo + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if cveErr != nil { + return cveErr + } + if len(repoPath) > 1 && repoPath != path { + cveCenter.PackName = repoPath + cveCenter.RepoName = issueTmp.OwnedComponent + models.UpdateVulnCenter(&cveCenter, "PackName", "RepoName") + issueTmp.Repo = repoPath + models.UpdateIssueTemplate(&issueTmp, "Repo") + path = repoPath + } + if issueHook.Issue.StateName == "已挂起" { + logs.Error("The current issue has been suspended and will not be processed, issueHook: ", issueHook) + issueTmp.Status = 5 + models.UpdateIssueTemplate(&issueTmp, "Status") + return errors.New("The current issue has been suspended and will not be processed") + } + if issueHook.Issue.StateName == "已拒绝" { + logs.Error("The current issue has been rejected and will not be processed, issueHook: ", issueHook) + issueTmp.Status = 4 + models.UpdateIssueTemplate(&issueTmp, "Status") + return errors.New("The current issue has been rejected and will not be processed") + } + owner, token := common.GetOwnerAndToken(cveCenter.OrganizationID) + issueTmp.StatusName = issueHook.Issue.StateName + logs.Info("Initiating issue status modification, sponsor: @", issueHook.Sender.UserName, ", Modify status: ", + issueHook.Issue.StateName, ", data: ", issueHook) + switch issueHook.State { + case IssueOpenState: + issueTmp.Status = 1 + cveCenter.IsExport = 0 + checkFunc := checkIssueAnalysisComplete + if cveCenter.OrganizationID == 2 { + checkFunc = checkGaussIssueAnalysisComplete + } else if cveCenter.OrganizationID == 3 { + checkFunc = checkSporeIssueAnalysisComplete + } else if cveCenter.OrganizationID == 4 { + checkFunc = checkLooKengIssueAnalysisComplete + } + _, _, ok := checkFunc(&issueTmp) + if ok { + issueTmp.IssueStatus = 3 + } else { + issueTmp.IssueStatus = 1 + } + issueTmp.IssueLabel = unFix + //issueTmp.IssueStatus = 1 + case IssueProgressState: + issueTmp.Status = 2 + cveCenter.IsExport = 0 + checkFunc := checkIssueAnalysisComplete + if cveCenter.OrganizationID == 2 { + checkFunc = checkGaussIssueAnalysisComplete + } else if cveCenter.OrganizationID == 3 { + checkFunc = checkSporeIssueAnalysisComplete + } else if cveCenter.OrganizationID == 4 { + checkFunc = checkLooKengIssueAnalysisComplete + } + _, _, ok := checkFunc(&issueTmp) + if ok { + issueTmp.IssueStatus = 3 + } else { + issueTmp.IssueStatus = 1 + } + issueTmp.IssueLabel = unFix + case IssueCloseState: + if issueTmp.Status == 3 { + // The issue has been closed and cannot be operated again + logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) + return errors.New("The issue has been closed and cannot be operated again") + } + if cveCenter.OrganizationID == 1 { + closeIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, &cveCenter) + } else { + otherCloseIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, path, &cveCenter) + } + case IssueRejectState: + issueTmp.Status = 4 + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = uNaffected + } + updateBool := updateTempAndCenter(issueTmp, cveCenter, token, owner) + if !updateBool { + return errors.New("handle issue state hook appear error maybe some step fail") + } + return nil +} + +// When the issue status is complete, verify whether the pr is associated +func VerifyIssueAsPr(issueTmp *models.IssueTemplate, cveCenter models.VulnCenter, + effectFlag bool, assignee, prSend string) bool { + sn := models.SecurityNotice{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} + secErr := sn.Read("cve_id", "cve_num") + if secErr != nil { + logs.Error("no data has been found, issueTmp: ", issueTmp) + return true + } + affectBranchsxList := make([]string, 0) + affectedBranchs := "" + path := cveCenter.PackName + affectProductList := make([]string, 0) + tmpAffectBranchsxList := make([]string, 0) + owner, token := common.GetOwnerAndToken(cveCenter.OrganizationID) + if cveCenter.OrganizationID == 4 { + affectedBranchs = beego.AppConfig.String("openlookeng::openlookeng_version") + affectBranchsxList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) + } else if cveCenter.OrganizationID == 3 { + affectedBranchs = beego.AppConfig.String("mindspore::mindspore_version") + affectBranchsxList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) + } else if cveCenter.OrganizationID == 2 { + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } + affectBranchsxList, _ = taskhandler.GetBranchesInfo(token, owner, path, cveCenter.OrganizationID) + } else { + affectedBranchs = beego.AppConfig.String("cve::affected_branchs") + path = issueTmp.Repo + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + } + + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + tmpAffectBranchsxList = strings.Split(sn.AffectProduct, "/") + } + if len(tmpAffectBranchsxList) > 0 { + sort.Strings(tmpAffectBranchsxList) + repSlice := common.SliceRemoveDup(tmpAffectBranchsxList) + for _, rep := range repSlice { + if len(rep.(string)) > 0 { + affectProductList = append(affectProductList, rep.(string)) + } + } + } + + if len(affectProductList) > 0 { + issueTmp.SaAuditFlag = 0 + branchMaps := make(map[string]bool) + for _, brands := range affectProductList { + brands = common.BranchVersionRep(brands) + if len(affectBranchsxList) > 0 { + keyBandList := []string{} + for _, affectBranch := range affectBranchsxList { + affectBranch = common.BranchVersionRep(affectBranch) + if affectBranch == brands { + // Query the repo that needs to submit an issue + if cveCenter.OrganizationID == 3 { + mdbt := models.MindSporeBrandTags{PackageName: path, Tags: affectBranch} + mtErr := models.QueryMindSporeBrandTags(&mdbt, "PackageName", "Tags") + if mtErr == nil { + keyBandList = append(keyBandList, mdbt.Brand) + } + } else if cveCenter.OrganizationID == 4 { + keyBandList = append(keyBandList, "master") + } else { + keyBandList = append(keyBandList, affectBranch) + } + } + } + if len(keyBandList) > 0 { + prList := getRepoIssueAllPR(keyBandList, token, owner, path, *issueTmp) + for _, brh := range keyBandList { + if len(prList) == 0 { + branchMaps[brh] = false + } else { + for _, prl := range prList { + if brh == prl.Branch { + branchMaps[brh] = prl.BrFlag + break + } + } + } + if _, ok := branchMaps[brh]; !ok { + branchMaps[brh] = false + } + } + } + } + } + if len(branchMaps) == 0 { + logs.Info("sn.AffectProduct: ", sn.AffectProduct, ",There is no branch to follow to associate with pr") + return true + } + brandStr := "" + //logs.Info("branchMaps===> ", branchMaps) + for brand, bv := range branchMaps { + if !bv { + logs.Error("brand: ", brand, ", pr is not related to issue, issueTmp: ", issueTmp) + brandStr = brandStr + brand + "/" + } + } + if brandStr != "" && len(brandStr) > 1 { + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, path, + cveCenter, *issueTmp) + if issueErr == nil { + commentBody := assignee + "\n" + + "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #" + issueTmp.IssueNum + "\n" + + "受影响分支: " + brandStr[:len(brandStr)-1] + "\n" + + "具体操作参考: " + "https://gitee.com/help/articles/4142" + "\n" + taskhandler.AddCommentToIssue(commentBody, issueTmp.IssueNum, owner, path, token) + content := issueTmp.Repo + " 仓库的CVE和安全问题的ISSUE,CVE编号: " + issueTmp.CveNum + + ",关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #" + issueTmp.IssueNum + + ",受影响分支: " + brandStr[:len(brandStr)-1] + + ",具体操作参考: " + "https://gitee.com/help/articles/4142." + taskhandler.SendPrivateLetters(token, content, prSend) + } + return false + } + } else { + if effectFlag && issueTmp.SaAuditFlag == 0 { + unaffectedBranchList := []string{} + if issueTmp.AffectedVersion != "" && len(issueTmp.AffectedVersion) > 1 { + unaffectedBranchList = paraAffectBrands(issueTmp.AffectedVersion) + } + branchStrs := "" + if len(unaffectedBranchList) > 0 { + for _, brands := range unaffectedBranchList { + if len(affectBranchsxList) > 0 { + for _, affectBranch := range affectBranchsxList { + if affectBranch == brands { + branchStrs = branchStrs + brands + "/" + } + } + } + } + } + if branchStrs != "" && len(branchStrs) > 1 { + branchStrs = branchStrs[:len(branchStrs)-1] + list, err := models.GetSecurityReviewerList() + if err != nil { + logs.Error("err: ", err, "\n, issueTmp: ", issueTmp) + issueTmp.SaAuditFlag = 1 + return true + } + if len(list) == 0 { + logs.Error("list is null, issueTemp: ", issueTmp) + issueTmp.SaAuditFlag = 1 + return true + } + anName := []string{} + content := fmt.Sprintf(CommentPrivateReview, issueTmp.Repo, issueTmp.CveNum) + for _, v := range list { + anName = append(anName, "@"+v.NameSpace+" ") + taskhandler.SendPrivateLetters(token, content, v.NameSpace) + } + if len(anName) > 0 { + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + assignee := strings.Join(anName, ",") + commentBody := assignee + "\n" + + "关闭issue前,请确认分支: " + branchStrs + ": 受影响/不受影响, 如受影响,请联系maintainer: @" + + issueTmp.Assignee + ", **进行处理后, 或者按照模板格式在评论区填写内容, 最后记得在评论区回复: /approve ,才能正常关闭issue.**" + taskhandler.AddCommentToIssue(commentBody, issueTmp.IssueNum, owner, issueTmp.Repo, token) + } + return false + } else { + issueTmp.SaAuditFlag = 1 + } + } else { + issueTmp.SaAuditFlag = 1 + } + } + } + return true +} + +func paraAffectBrands(affectedVersion string) (unaffectedBranchList []string) { + brandsGroup := strings.Split(affectedVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams != "受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } else { + brandList = strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams != "受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + if len(brandList) == 1 { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + return unaffectedBranchList +} + +func paraAffectBrandBool(affectedVersion string) bool { + unaffectedBranchList := make([]string, 0) + brandsGroup := strings.Split(affectedVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "受影响" || prams == "不受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } else { + brandList = strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "受影响" || prams == "不受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + } + } + if len(unaffectedBranchList) > 0 { + return true + } + return false +} + +func getPRRelatedBrandsAllIssue(token, owner, repo string, num int, issueNum string) bool { + issueFlag := false + url := fmt.Sprintf(`https://gitee.com/api/v5/repos/%s/%s/pulls/%v/issues`, owner, repo, num) + pageSize := 20 + pageCount := 1 + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error(err) + return false + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("per_page", strconv.Itoa(pageSize)) + for { + q.Del("page") + q.Add("page", strconv.Itoa(pageCount)) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error(err) + break + } + if resp.StatusCode == http.StatusOK { + var il []models.HookIssue + read, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + logs.Error(err) + break + } + err = json.Unmarshal(read, &il) + if err != nil { + logs.Error(err) + break + } + for _, v := range il { + d, ok := isLegallyIssue(v) + if ok { + if issueNum == d.Number { + issueFlag = true + break + } + } + } + if len(il) < pageSize { + break + } + pageCount++ + } else { + resp.Body.Close() + break + } + } + return issueFlag +} + +// Verify that the current issue meets the requirements +func isLegallyIssue(i models.HookIssue) (pri models.PullRequestIssue, ok bool) { + if i.IssueType != CIssueType || (i.State != "closed" && i.State != "已完成") { + return + } + tt := strings.Trim(i.Title, " ") + regCveNum := regexp.MustCompile(`(?mi)CVE-[\d]{1,}-([\d]{1,})$`) + sm := util.RegexpCveNumber.FindAllStringSubmatch(i.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + val := sm[0][1] + tt = util.GetCveNumber(util.TrimString(val)) + if tt != "" && regCveNum.Match([]byte(tt)) { + ok = true + } + } + if ok { + pri.Id = i.Id + pri.Number = i.Number + pri.CveNumber = tt + pri.Repo = i.Repository.Path + } + return +} + +// Get the pr associated with a single repo +func getRepoIssueAllPR(affectBranch []string, token, owner, repo string, isTemp models.IssueTemplate) (prList []models.PullRequestIssue) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests", owner, isTemp.IssueNum) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error("NewRequest, url: ", url, ",err: ", err) + return + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("repo", repo) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error("DefaultClient, url: ", url, ",err: ", err) + return + } + if resp.StatusCode == http.StatusOK { + issuePr := make([]map[string]interface{}, 0) + read, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, url: ", url, ",err: ", err) + return + } + resp.Body.Close() + //logs.Info("getRepoIssueAllPR, body ===> ", read) + err = json.Unmarshal(read, &issuePr) + if err != nil { + logs.Error("Unmarshal, url: ", url, ",err: ", err) + return + } + logs.Info("issuePr: ", issuePr) + for _, brh := range affectBranch { + pr := models.PullRequestIssue{} + pr.Branch = brh + pr.BrFlag = false + for _, v := range issuePr { + if _, ok := v["id"]; !ok { + continue + } + if v["state"].(string) == "merged" && v["mergeable"].(bool) { + if v["base"].(map[string]interface{})["label"].(string) == brh { + //if v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo { + // if v["base"].(map[string]interface{})["repo"].(map[string]interface{})["namespace"].(map[string]interface{})["path"] == owner { + // pr.Id = int64(v["id"].(float64)) + // pr.Number = isTemp.IssueNum + // pr.CveNumber = isTemp.CveNum + // pr.Repo = repo + // prList = append(prList, pr) + // } + //} + pr.Id = int64(v["id"].(float64)) + pr.Number = isTemp.IssueNum + pr.CveNumber = isTemp.CveNum + pr.Repo = repo + pr.BrFlag = true + break + } + } + } + prList = append(prList, pr) + } + } else { + resp.Body.Close() + } + return +} + +func isNormalCloseIssue(cveID int64, issueState int8) bool { + if issueState == 1 { + return false + } + score, err := models.QueryIssueScore(cveID) + if err != nil { + logs.Error("err: ", err, "\n, cveID: ", cveID) + return false + } + if score.Ostatus != 3 { + return false + } + return true +} + +// update data +func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCenter, token, owner string) bool { + appearErr := 0 + affectBrandFlag := false + uNaffected := beego.AppConfig.String("labeUnaffected") + labelFixed := beego.AppConfig.String("labelFixed") + labelUnFix := beego.AppConfig.String("labelUnFix") + labeAbiChanged := beego.AppConfig.String("labeAbiChanged") + path := cveCenter.PackName + sn := models.SecurityNotice{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} + snErr := sn.Read("cve_id", "cve_num") + if snErr != nil { + logs.Error("err: ", snErr) + } else { + if issueTmp.Status > 2 { + affectBranchsxList := make([]string, 0) + affectedBranchs := "" + if cveCenter.OrganizationID == 4 { + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + } else if cveCenter.OrganizationID == 3 { + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + tmpTagList := make([]string, 0) + affectProductSlice := strings.Split(sn.AffectProduct, "/") + for _, tags := range affectProductSlice { + mdbt := models.MindSporeBrandTags{PackageName: path, Tags: tags} + mtErr := models.QueryMindSporeBrandTags(&mdbt, "PackageName", "Tags") + if mtErr == nil { + tmpTagList = append(tmpTagList, mdbt.Brand) + } + } + if len(tmpTagList) > 0 { + sn.AffectProduct = strings.Join(tmpTagList, "/") + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + } else { + sn.AffectProduct = "" + } + } + } else if cveCenter.OrganizationID == 2 { + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + } else { + affectedBranchs = beego.AppConfig.String("cve::affected_branchs") + } + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + affectProductList := strings.Split(sn.AffectProduct, "/") + if len(affectProductList) > 0 { + for _, brands := range affectProductList { + if len(affectBranchsxList) > 0 { + for _, affectBranch := range affectBranchsxList { + if affectBranch == brands { + affectBrandFlag = true + break + } + } + } + if affectBrandFlag { + break + } + } + } + } + } + switch issueTmp.IssueStatus { + case 2: + sn.AffectStatus = "Fixed" + case 6: + sn.AffectStatus = "UnAffected" + default: + sn.AffectStatus = "UnFixed" + } + err := sn.Update("affect_status") + if err != nil { + appearErr++ + logs.Error("err: ", err, ", issueTmp.IssueStatus: ", issueTmp.IssueStatus) + } + } + cveCenter.CveLevel = models.OpenEulerScoreProc(issueTmp.NVDScore) + if len(cveCenter.CveVersion) > 0 && cveCenter.CveVersion[0] == ',' { + cveCenter.CveVersion = cveCenter.CveVersion[1:] + } + update := models.UpdateVulnCenter(&cveCenter, "is_export", "cve_level", "CveVersion") + if !update { + logs.Error("update vulnCenter fail ") + appearErr += 1 + } + if issueTmp.StatusName == "closed" || issueTmp.Status == 3 { + if affectBrandFlag { + issueTmp.IssueLabel = labelFixed + } else { + issueTmp.IssueLabel = uNaffected + } + } else { + issueTmp.IssueLabel = labelUnFix + } + + labelSlice := make([]string, 0) + labelSlice = taskhandler.AddLabelValue(token, path, issueTmp.IssueNum, owner, issueTmp.IssueLabel, 1) + if len(labelSlice) > 0 { + labelStr := strings.Join(labelSlice, ",") + if issueTmp.Status == 3 { + if affectBrandFlag { + if strings.Contains(labelStr, uNaffected) { + labelSlice = common.DeleteSliceValue(labelSlice, uNaffected) + } + if !strings.Contains(labelStr, labelFixed) { + labelSlice = append(labelSlice, labelFixed) + } + } else { + if strings.Contains(labelStr, labelFixed) { + labelSlice = common.DeleteSliceValue(labelSlice, labelFixed) + } + if !strings.Contains(labelStr, uNaffected) { + labelSlice = append(labelSlice, uNaffected) + } + } + if strings.Contains(labelStr, labelUnFix) { + labelSlice = common.DeleteSliceValue(labelSlice, labelUnFix) + } + } else { + if !strings.Contains(labelStr, labelUnFix) { + labelSlice = append(labelSlice, labelUnFix) + } + } + } else { + if issueTmp.Status == 3 { + if affectBrandFlag { + labelSlice = append(labelSlice, labelFixed) + } else { + labelSlice = append(labelSlice, uNaffected) + } + } else { + labelSlice = append(labelSlice, labelUnFix) + } + } + issueTmp.IssueLabel = strings.Join(labelSlice, ",") + if issueTmp.Status == 3 && cveCenter.OrganizationID == 1 && + AbiAffectedVersionBool(issueTmp.AbiVersion) && !strings.Contains(issueTmp.IssueLabel, labeAbiChanged) { + issueTmp.IssueLabel = issueTmp.IssueLabel + "," + labeAbiChanged + } + update = taskhandler.UpdateIssueLabels(token, path, issueTmp.IssueNum, owner, issueTmp.IssueLabel) + if !update { + logs.Error("update gitee issue label fail ,", issueTmp.IssueNum, issueTmp.IssueLabel) + appearErr++ + } + issueTmp.CveLevel = models.OpenEulerScoreProc(issueTmp.OpenEulerScore) + tpErr := models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", + "status_name", "issue_label", "mt_audit_flag", "sa_audit_flag", "cve_level") + if tpErr != nil { + logs.Error("tpErr: ", tpErr) + appearErr += 1 + } + if appearErr > 0 { + logs.Error("handle issue state hook appear error maybe some step "+ + "fail, appearErr: ", appearErr, ",issuetmp: ", issueTmp) + return false + } + return true +} + +func AbiAffectedVersionBool(abiVersion string) bool { + abiVersionList := []string{} + brandsGroup := strings.Split(abiVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "是" { + abiVersionList = append(abiVersionList, brandList[0]) + break + } + } else { + brandList = strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "是" { + abiVersionList = append(abiVersionList, brandList[0]) + break + } + } + } + } + } + if len(abiVersionList) > 0 { + return true + } + return false +} + +func openEulerScoreReview(issueTmp *models.IssueTemplate, cuAccount, owner, token string) bool { + approveFlag := true + if isReviewer(cuAccount) { + issueTmp.OpAuditFlag = 1 + approveFlag = false + err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) + if err != nil { + logs.Error("1.err: ", err, ",issueTmp: ", issueTmp) + } else { + err = models.UpdateIssueTemplate(issueTmp, "op_audit_flag") + if err != nil { + logs.Error("2.err: ", err, ",issueTmp: ", issueTmp) + } + taskhandler.AddCommentToIssue(fmt.Sprintf(ReviewApproveScore, issueTmp.Assignee, cuAccount), + issueTmp.IssueNum, owner, issueTmp.Repo, token) + } + } else { + taskhandler.AddCommentToIssue(fmt.Sprintf(CommentReviewRemindMaintainer, cuAccount), + issueTmp.IssueNum, owner, issueTmp.Repo, token) + } + return approveFlag +} + +func gaussMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, + unfixed, path string, cveCenter models.VulnCenter) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, + issueTmp.OwnedVersion, cveCenter.OrganizationID) + if len(unFixList) > 0 { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } + if _, tb, ok := checkGaussIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } else { + if !isGaussReviewer(cuAccount) && issueTmp.Assignee != cuAccount { + logs.Error("Invalid user review, cuAccount: ", cuAccount) + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v maintainer具有通过(/approve或者/close)关闭issue, 否则请通过issue页面按钮关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + return + } + issueTmp.IssueLabel = unfixed + issueTmp.StatusName = "open" + issueTmp.Status = 1 + assignee := "@" + issueTmp.Assignee + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter, false, + assignee, cuAccount) + if issuePrFlag { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容, cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("Initiate an issue to close,issuetmp: ", issueTmp) + } else { + logs.Error("Issue closing operation failed, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + //issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") + issueTmp.Repo = gaussIssuePath + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } + } +} + +func looKengMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, + unfixed, path string, cveCenter models.VulnCenter) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, + issueTmp.OwnedVersion, cveCenter.OrganizationID) + if len(unFixList) > 0 { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } + if _, tb, ok := checkLooKengIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } else { + if !isLooKengReviewer(cuAccount) && issueTmp.Assignee != cuAccount { + logs.Error("Invalid user review, cuAccount: ", cuAccount) + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v maintainer具有通过(/approve或者/close)关闭issue, 否则请通过issue页面按钮关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + return + } + issueTmp.IssueLabel = unfixed + issueTmp.StatusName = "open" + issueTmp.Status = 1 + assignee := "@" + issueTmp.Assignee + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter, false, assignee, cuAccount) + if issuePrFlag { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容, cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("Initiate an issue to close,issuetmp: ", issueTmp) + } else { + logs.Error("Issue closing operation failed, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + //issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } + } +} + +func sporeMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, + unfixed, path string, cveCenter models.VulnCenter) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, + issueTmp.OwnedVersion, cveCenter.OrganizationID) + if len(unFixList) > 0 { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } + if _, tb, ok := checkSporeIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } else { + if !isMindSporeReviewer(cuAccount) && issueTmp.Assignee != cuAccount { + logs.Error("Invalid user review, cuAccount: ", cuAccount) + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v maintainer具有通过(/approve或者/close)关闭issue, 否则请通过issue页面按钮关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + return + } + issueTmp.IssueLabel = unfixed + issueTmp.StatusName = "open" + issueTmp.Status = 1 + assignee := "@" + issueTmp.Assignee + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter, false, assignee, cuAccount) + if issuePrFlag { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容, cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("Initiate an issue to close,issuetmp: ", issueTmp) + } else { + logs.Error("Issue closing operation failed, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + //issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } + } +} + +func maintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, + unfixed string, organizationID int8) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, + issueTmp.Repo, issueTmp.OwnedVersion, organizationID) + if len(unFixList) > 0 { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + return + } + if _, tb, ok := checkIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + return + } else { + issueTmp.MtAuditFlag = 1 + err := models.UpdateIssueTemplate(issueTmp, "mt_audit_flag") + if err != nil { + logs.Error("upErr: ", err, "issueTmp: ", issueTmp) + return + } + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} + err = models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if err != nil { + return + } + issueTmp.IssueLabel = unfixed + issueTmp.StatusName = "open" + assignee := getMaintainer(issueTmp.Repo, cuAccount, issueTmp.Assignee) + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter, false, assignee, cuAccount) + if issuePrFlag { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容, cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, issueTmp.Repo, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("Initiate an issue to close, issuetmp: ", issueTmp) + } else { + logs.Error("Issue closing operation failed, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + //issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } + return + } +} + +func securityApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, + fixed, unfixed string, organizationID int8) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, + issueTmp.Repo, issueTmp.OwnedVersion, organizationID) + if len(unFixList) > 0 { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + return + } + if _, tb, ok := checkIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + return + } else { + if !isReviewer(cuAccount) && issueTmp.Assignee != cuAccount { + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v maintainer具有通过(/approve或者/close)关闭issue, 否则请通过issue页面按钮关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, issueTmp.Repo, token) + return + } + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} + err := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if err != nil { + return + } + issueTmp.IssueLabel = unfixed + issueTmp.StatusName = "open" + assignee := getMaintainer(issueTmp.Repo, cuAccount, issueTmp.Assignee) + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter, false, assignee, cuAccount) + if issuePrFlag { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容,cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, issueTmp.Repo, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("Initiate an issue to close, issuetmp: ", issueTmp) + } else { + logs.Error("issue close operation failed, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } + return + } +} + +func handleIssueComment(payload models.CommentPayload) { + if payload.Issue == nil || payload.Comment == nil { + return + } + if payload.Comment.User == nil { + return + } + // The default timeout for receiving hooks + logs.Info("payload.Comment: ", payload.Comment, ", Number: ", payload.Issue.Number, "id: ", payload.Issue.Id) + issueNum := payload.Issue.Number //issue number string + issueId := payload.Issue.Id // issue id int64 + cBody := payload.Comment.Body //Comment subject + cuAccount := payload.Comment.User.UserName //gitee domain address + repoPath := payload.Issue.Repository.Path + cmdRej := beego.AppConfig.DefaultString("rejectCmd", "/reject") + cmdApe := beego.AppConfig.DefaultString("approveCmd", "/approve") + cmdClose := beego.AppConfig.DefaultString("closeCmd", "/close") + cmdError := beego.AppConfig.DefaultString("errorCmd", "/error") + cmdAutoPr := beego.AppConfig.String("autoPrCmd") + cmdGetCve := beego.AppConfig.String("getCveCmd") + cmdFeedBack := beego.AppConfig.String("feedBackCmd") + if issueNum == "" || cuAccount == "" || cBody == "" { + logs.Error("Data has null values: issueNum, cuAccount, cBody: ", issueNum, cuAccount, cBody) + return + } + // Ignore this comment + botCuAccountStr := beego.AppConfig.String("cve::bot_cu_account") + botCuAccountList := strings.Split(botCuAccountStr, ",") + if len(botCuAccountList) > 0 { + for _, botCu := range botCuAccountList { + if cuAccount == botCu { + logs.Error(cuAccount, ", Ignore this comment") + return + } + } + } + issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} + err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if err != nil { + logs.Error("GetErr: ", err, ",issueTmp: ", issueTmp) + return + } + agencyPram := AgencyPrams{} + fixed := beego.AppConfig.String("labelFixed") + unfixed := beego.AppConfig.String("labelUnFix") + path := issueTmp.Repo + vc := models.VulnCenter{CveId: issueTmp.CveId} + vcErr := models.GetVulnCenterByCid(&vc, "CveId") + if vcErr != nil { + logs.Error("GetVulnCenterByCid, vcErr: ", vcErr, ",CveId: ", issueTmp.CveId) + return + } + owner, accessToken := common.GetOwnerAndToken(vc.OrganizationID) + if vc.OrganizationID == 2 { + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } + cBody = strings.ReplaceAll(cBody, util.KwOpenGaussScore, util.KwOpenEulerScore) + } else if vc.OrganizationID == 3 { + // Query the repo that needs to submit an issue + cBody = strings.ReplaceAll(cBody, util.KwMindSporeScore, util.KwOpenEulerScore) + } else if vc.OrganizationID == 4 { + cBody = strings.ReplaceAll(cBody, util.KwLooKengScore, util.KwOpenEulerScore) + } + if len(repoPath) > 1 && repoPath != path { + vc.PackName = repoPath + vc.RepoName = issueTmp.OwnedComponent + models.UpdateVulnCenter(&vc, "PackName", "RepoName") + issueTmp.Repo = repoPath + models.UpdateIssueTemplate(&issueTmp, "Repo") + path = repoPath + } + if payload.Issue.StateName == "已挂起" { + logs.Error("The current issue has been suspended and will not be processed, payload: ", payload) + issueTmp.Status = 5 + models.UpdateIssueTemplate(&issueTmp, "Status") + taskhandler.AddCommentToIssue(fmt.Sprintf(CommentRejectedState, cuAccount, payload.Issue.StateName), + issueTmp.IssueNum, owner, path, accessToken) + return + } + if payload.Issue.StateName == "已拒绝" { + logs.Error("The current issue has been rejected and will not be processed, payload: ", payload) + issueTmp.Status = 4 + models.UpdateIssueTemplate(&issueTmp, "Status") + taskhandler.AddCommentToIssue(fmt.Sprintf(CommentRejectedState, cuAccount, payload.Issue.StateName), + issueTmp.IssueNum, owner, path, accessToken) + return + } + if strings.HasPrefix(cBody, cmdRej) { + //Review rejected Add comment @Analyst + if !isReviewer(cuAccount) { + return + } + if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.NVDScore > 0 && + issueTmp.OpenEulerScore > 0 { + err = changeOpenEulerScoreStatus(issueTmp.CveId, 2) + if err != nil { + logs.Error("changeOpenEulerScoreStatus, 1.err:", err) + } + issueTmp.OpAuditFlag = 2 + err = models.UpdateIssueTemplate(&issueTmp, "op_audit_flag") + if err != nil { + logs.Error("UpdateIssueTemplate, 2.err:", err) + } + taskhandler.AddCommentToIssue(fmt.Sprintf(ReviewRejectScore, issueTmp.Assignee, cuAccount), + issueTmp.IssueNum, owner, path, accessToken) + } + } else if strings.HasPrefix(cBody, cmdApe) || strings.HasPrefix(cBody, cmdClose) { + if issueTmp.Status == 3 { + // The issue has been closed and cannot be operated again + logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) + return + } + if vc.OrganizationID == 4 { + comLock.Lock() + looKengMaintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, path, vc) + comLock.Unlock() + } else if vc.OrganizationID == 3 { + comLock.Lock() + sporeMaintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, path, vc) + comLock.Unlock() + } else if vc.OrganizationID == 2 { + comLock.Lock() + gaussMaintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, path, vc) + comLock.Unlock() + } else { + approveFlag := true + if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.NVDScore > 0 && + issueTmp.OpAuditFlag != 1 && issueTmp.OpenEulerScore > 0 { + //Approved to modify the rating status + approveFlag = openEulerScoreReview(&issueTmp, cuAccount, owner, accessToken) + } + if approveFlag { + mtAuditFlag := false + // Analysis command belongs to the time period + maintainerList, mainOk := models.QueryRepoAllMaintainer(issueTmp.Repo) + if mainOk && len(maintainerList) > 0 { + for _, v := range maintainerList { + if util.TrimString(v.MemberName) == cuAccount { + mtAuditFlag = true + break + } + } + } + if mtAuditFlag { + comLock.Lock() + maintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, vc.OrganizationID) + comLock.Unlock() + } else { + comLock.Lock() + securityApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, vc.OrganizationID) + comLock.Unlock() + } + } + } + } else if strings.HasPrefix(cBody, cmdError) { + c := strings.TrimPrefix(cBody, cmdError) + c = strings.TrimSpace(c) + issueTmp.ErrorDescription = c + issueTmp.UpdateTime = time.Now() + //update issue tpl + err = models.UpdateIssueTemplate(&issueTmp, "error_description", "update_time") + if err != nil { + logs.Error("updateIssueErr: ", err, "issueTmp: ", issueTmp) + return + } + } else if strings.HasPrefix(cBody, cmdAutoPr) { + if vc.OrganizationID == 1 { + PostCveAgency(issueTmp, agencyPram) + } + } else if strings.HasPrefix(cBody, cmdFeedBack) { + if vc.OrganizationID == 1 { + bodyList := strings.Split(cBody, " ") + if len(bodyList) > 2 { + agencyPram.CveUrl = bodyList[1] + agencyPram.PatchUrl = bodyList[2] + } + PostCveAgency(issueTmp, agencyPram) + } + } else if strings.HasPrefix(cBody, cmdGetCve) { + PostTriggerGetCve(issueTmp, owner, accessToken, cuAccount) + } else { + if payload.Issue.State == "closed" || payload.Issue.State == "rejected" || + payload.Issue.State == "已完成" || payload.Issue.State == "已拒绝" { + logs.Error("Cannot edit comment, value: ", payload.Issue) + return + } + cBody = strings.ReplaceAll(cBody, "：", ":") + comLock.Lock() + analysisComment(owner, accessToken, path, cuAccount, cBody, &payload, issueTmp, vc.OrganizationID) + comLock.Unlock() + } +} + +func PostCveAgency(issueTmp models.IssueTemplate, agencyPram AgencyPrams) { + // Trigger third-party data generation + agencyUrl := beego.AppConfig.String("cveagency::url") + agencyReq := make(map[string]interface{}, 0) + agencyReq["CveNum"] = issueTmp.CveNum + agencyReq["PackName"] = issueTmp.OwnedComponent + agencyReq["PackVersion"] = issueTmp.OwnedVersion + agencyReq["CveUrl"] = agencyPram.CveUrl + agencyReq["PatchUrl"] = agencyPram.PatchUrl + if len(issueTmp.AffectedVersion) > 0 { + versionList := strings.Split(issueTmp.AffectedVersion, ",") + if len(versionList) > 0 { + vSlice := []string{} + for _, vl := range versionList { + vs := strings.Split(vl, ":") + if len(vs) > 0 { + vSlice = append(vSlice, vs[0]) + } + } + if len(vSlice) > 0 { + agencyReq["AffectBranch"] = strings.Join(vSlice, " ") + } + } + } else { + agencyReq["AffectBranch"] = "" + } + agencyReq["IssueNumber"] = issueTmp.IssueNum + util.HTTPPostCom(agencyReq, agencyUrl) +} + +func PostTriggerGetCve(issueTmp models.IssueTemplate, owner, accessToken, cuAccount string) { + // Trigger third-party data generation + cc := "" + if len(issueTmp.CveNum) > 0 { + resp := tabletask.PullCve(issueTmp.CveNum) + if resp == 1 { + cc = fmt.Sprintf(CommentGetNvdCveSuccess, cuAccount) + } else { + cc = fmt.Sprintf(CommentGetNvdCveFailed, cuAccount) + } + if len(cc) > 1 { + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, accessToken) + } + vc := models.VulnCenter{CveId: issueTmp.CveId} + vcErr := models.GetVulnCenterByCid(&vc, "CveId") + if vcErr != nil { + logs.Error("GetVulnCenterByCid, vcErr: ", vcErr, ",CveId: ", issueTmp.CveId) + return + } + models.UpdateIssueStatus(vc, 1) + issueErr := task.SyncCreateIssue() + if issueErr != nil { + logs.Error("SyncCveAndIssue, SyncCreateIssue, err: ", issueErr) + } + } +} + +func isReviewer(path string) bool { + sr := models.SecurityReviewer{NameSpace: path} + return sr.Read("name_space") +} + +func isGaussReviewer(path string) bool { + sr := models.OpenGaussSecurityReviewer{NameSpace: path} + return sr.Read("name_space") +} + +func isMindSporeReviewer(path string) bool { + sr := models.MindSporeSecurityReviewer{NameSpace: path} + return sr.Read("name_space") +} + +func isLooKengReviewer(path string) bool { + sr := models.OpenLookengSecurityReviewer{NameSpace: path} + return sr.Read("name_space") +} + +func analysisComment(owner, accessToken, path string, cuAccount string, cBody string, + payload *models.CommentPayload, issueTmp models.IssueTemplate, OrganizationID int8) { + if issueTmp.Status == 3 { + // The issue has been closed and cannot be operated again + logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) + return + } + canVerfy := false + issueTmp.MtAuditFlag = 1 + //is Analyst comment and content start with '/analysis' + vMap := util.ExtractCommentAnalysisAllValue(cBody, OrganizationID) + if len(vMap) > 0 { + canVerfy = true + cols := make([]string, 0) + for k, v := range vMap { + switch k { + case "cve_analysis": + if v != "" && len(v) > 1 { + issueTmp.CveAnalysis = common.DeletePreAndSufSpace(v) + cols = append(cols, k) + } + case "principle_analysis": + issueTmp.PrincipleAnalysis = v + cols = append(cols, k) + case "openeuler_score": + fv, err := strconv.ParseFloat(v, 64) + if err == nil && fv > 0 { + if issueTmp.OpenEulerScore > 0 && issueTmp.OpenEulerScore != fv { + issueTmp.OpAuditFlag = 0 + cols = append(cols, "op_audit_flag") + } + issueTmp.OpenEulerScore = fv + cols = append(cols, k) + } + case "openeuler_vector": + if v != "" && len(v) > 1 { + issueTmp.OpenEulerVector = common.DeletePreAndSufSpace(v) + cols = append(cols, k) + } + case "affected_version": + if v != "" && len(v) > 1 { + if paraAffectBrandBool(v) { + issueTmp.AffectedVersion = v + cols = append(cols, k) + } + } + case "abi_version": + if v != "" && len(v) > 1 { + if taskhandler.ParaAffectAbiBool(v) { + issueTmp.AbiVersion = v + cols = append(cols, k) + } + } + case "solution": + issueTmp.Solution = v + cols = append(cols, k) + } + } + if len(cols) > 0 { + cols = append(cols, "mt_audit_flag") + err := models.UpdateIssueTemplate(&issueTmp, cols...) + if err != nil { + logs.Error("uperr: ", err, ",issueTmp: ", issueTmp) + } else { + if _, ok := vMap["openeuler_vector"]; ok { + err := saveVectorData(vMap["openeuler_vector"], issueTmp.CveId) + if err != nil { + logs.Error("saveVectorData, err: ", err) + } + } + if _, ok := vMap["openeuler_score"]; ok { + //更新分数到 score + score, err := models.QueryIssueScore(issueTmp.CveId) + if err != nil { + logs.Error("queryErr: ", err, "cveId: ", issueTmp.CveId) + } else { + score.OpenEulerScore = issueTmp.OpenEulerScore + score.Ostatus = 1 + err := models.UpdateScore(&score, "openeuler_score", "o_score_status") + if err != nil { + logs.Error("upErr: ", err, ",UpdateScore, score: ", score) + } + } + } + } + // update gitee issue + commentUpdateIssue(issueTmp, owner, accessToken, path) + } + if _, ok := vMap["issue_package"]; ok { + // handle comment package + err := handleCommentPackage(vMap["issue_package"], issueTmp.CveId) + if err != nil { + logs.Error("handleCommentPackage, err: ", err) + } + } + } + if canVerfy { + //Check whether the data is legal + checkFunc := checkIssueAnalysisComplete + if OrganizationID == 2 { + checkFunc = checkGaussIssueAnalysisComplete + } else if OrganizationID == 3 { + checkFunc = checkSporeIssueAnalysisComplete + } else if OrganizationID == 4 { + checkFunc = checkLooKengIssueAnalysisComplete + } + if msg, tb, ok := checkFunc(&issueTmp); !ok { + //send comment to issue + issueTmp.IssueStatus = 1 + err := models.UpdateIssueTemplate(&issueTmp, "issue_status") + if err != nil { + logs.Error("UpdateIssueTemplate, upErr: ", err, ",issueTmp: ", issueTmp) + } + assignee := "" + if cuAccount != "" && len(cuAccount) > 1 { + assignee = cuAccount + } else { + assignee = issueTmp.Assignee + } + msg = fmt.Sprintf(CommentAnalysisCplTpl, assignee, msg) + taskhandler.AddCommentToIssue(msg, issueTmp.IssueNum, owner, path, accessToken) + } else { + if OrganizationID == 1 { + //1. change issue status + issueTmp.IssueStatus = 3 + //2. Are the cvsScore and openEuler score equal .If not equal, notify the auditor to review . + var na string + if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.OpenEulerScore > 0 && + issueTmp.NVDScore > 0 && issueTmp.OpAuditFlag == 0 { + na = "\n**因OpenEulerScore与NvdScore不一致,分析内容需审核,请等待安全组审核!**" + //Notify the responsible person for review + notifyAuditorReview(payload, issueTmp) + } else { + if issueTmp.MtAuditFlag == 0 { + maintainerList, mainOk := models.QueryRepoAllMaintainer(issueTmp.Repo) + assList := []string{} + if mainOk && len(maintainerList) > 0 { + for _, v := range maintainerList { + assList = append(assList, "@"+v.MemberName+" ") + content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v, "+ + "已经完成了模板填写,需要您对填写的内容进行审核,审核通过才能进行后续操作.", issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(accessToken, content, v.MemberName) + } + } + assignee := "" + if len(assList) > 0 { + assignee = strings.Join(assList, ",") + } else { + assignee = "@" + issueTmp.Assignee + } + na = "\n**请确认模板分析内容的准确性与完整性, 确认无误后,请在评论区输入: /approve, 否则无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) + } else { + na = "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" + cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) + } + // change score status + err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) + if err != nil { + logs.Error("changeOpenEulerScoreStatus, err: ", err, ",issueTmp: ", issueTmp) + } + } + } else { + na := "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" + cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) + // change score status + err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) + if err != nil { + logs.Error("changeOpenEulerScoreStatus, err: ", err, ",issueTmp: ", issueTmp) + } + } + } + err := models.UpdateIssueTemplate(&issueTmp, "issue_status", "mt_audit_flag") + if err != nil { + logs.Error("UpdateIssueTemplate, updErr: ", err, ",issueTmp: ", issueTmp) + } + } +} + +func notifyAuditorReview(payload *models.CommentPayload, issueTmp models.IssueTemplate) { + //Notify the responsible person for review + list, err := models.GetSecurityReviewerList() + if err != nil { + logs.Error("GetSecurityReviewerList, err: ", err) + return + } + if len(list) == 0 { + return + } + accessToken := os.Getenv("GITEE_TOKEN") + content := fmt.Sprintf(ReviewPrivateLettersTpl, + payload.Issue.Title, payload.Issue.HtmlUrl, issueTmp.NVDScore, issueTmp.OpenEulerScore) + owner := beego.AppConfig.String("gitee::owner") + //path := beego.AppConfig.String("gitee::path") + path := issueTmp.Repo + ns := make([]string, len(list)) + for k, v := range list { + ns[k] = "@" + v.NameSpace + " " + taskhandler.SendPrivateLetters(accessToken, content, v.NameSpace) + //add @comment + } + msg := fmt.Sprintf(CommentReviewTpl, strings.Join(ns, ",")) + taskhandler.AddCommentToIssue(msg, issueTmp.IssueNum, owner, path, accessToken) +} + +func changeOpenEulerScoreStatus(cveID int64, status int8) error { + score, err := models.QueryIssueScore(cveID) + if err != nil { + return err + } + score.Ostatus = status + err = models.UpdateScore(&score, "o_score_status") + return err +} + +func checkIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 15) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + return + } + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affectx := range affectedVersionArry { + affect := common.BranchVersionRep(affectx) + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + return + } + if versionfFlag { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } else { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openEulerScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openEulerVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + msg = fmt.Sprintf("2.openEulerScore=> 没有填写或正确填写(0-10)") + ok = false + return + } + tbContent[3] = "已分析" + tbContent[4] = "2.openEulerScore" + tbContent[5] = i.OpenEulerScore + + if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { + msg = fmt.Sprintf("2.openEulerVector=> 没有正确填写") + ok = false + return + } + tbContent[6] = "已分析" + tbContent[7] = "2.openEulerVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + if i.AbiVersion != "" { + versionAbiFlag := true + abiVersionArry := strings.Split(i.AbiVersion, ",") + if len(abiVersionArry) > 0 { + for _, affectx := range abiVersionArry { + affect := common.BranchVersionRep(affectx) + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "是" || versionArry[1] == "否" { + continue + } else { + versionAbiFlag = false + break + } + } else { + versionAbiFlag = false + break + } + } + } + if !versionAbiFlag { + msg = fmt.Sprintf("4.修复是否涉及abi变化(是/否)=> 没有分析或未按正确格式填写:%v", i.AbiVersion) + ok = false + return + } + if versionAbiFlag { + tbContent[12] = "已分析" + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = util.TrimStringNR(i.AbiVersion) + } else { + tbContent[12] = "待分析" + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = util.TrimStringNR(i.AbiVersion) + } + } else { + tbContent[12] = "已分析" + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = "" + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkGaussIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + return + } + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + return + } + if versionfFlag { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } else { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openGaussScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openGaussVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + msg = fmt.Sprintf("2.openGaussScore=> 没有填写或正确填写(0-10)") + ok = false + return + } + tbContent[3] = "已分析" + tbContent[4] = "2.openGaussScore" + tbContent[5] = i.OpenEulerScore + + if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { + msg = fmt.Sprintf("2.openGaussVector=> 没有正确填写") + ok = false + return + } + tbContent[6] = "已分析" + tbContent[7] = "2.openGaussVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkSporeIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + return + } + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + return + } + if versionfFlag { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } else { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + msg = fmt.Sprintf("2.MindSporeScore=> 没有填写或正确填写(0-10)") + ok = false + return + } + tbContent[3] = "已分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = i.OpenEulerScore + if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { + msg = fmt.Sprintf("2.MindSporeVector=> 没有正确填写") + ok = false + return + } + tbContent[6] = "已分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkLooKengIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + return + } + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + return + } + if versionfFlag { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } else { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + msg = fmt.Sprintf("2.openLooKengScore=> 没有填写或正确填写(0-10)") + ok = false + return + } + tbContent[3] = "已分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = i.OpenEulerScore + if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { + msg = fmt.Sprintf("2.openLooKengVector=> 没有正确填写") + ok = false + return + } + tbContent[6] = "已分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 15) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { + tbContent[0] = "待分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + } else { + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + } + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affectx := range affectedVersionArry { + affect := common.BranchVersionRep(affectx) + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openEulerScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openEulerVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + tbContent[3] = "待分析" + tbContent[4] = "2.openEulerScore" + tbContent[5] = fmt.Sprintf("2.openEulerScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openEulerScore=> 没有填写或正确填写(0-10)") + ok = false + } else { + tbContent[3] = "已分析" + tbContent[4] = "2.openEulerScore" + tbContent[5] = i.OpenEulerScore + } + if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { + tbContent[6] = "待分析" + tbContent[7] = "2.openEulerVector" + tbContent[8] = fmt.Sprintf("2.openEulerVector=> 没有正确填写") + msg = fmt.Sprintf("2.openEulerVector=> 没有正确填写") + ok = false + } else { + tbContent[6] = "已分析" + tbContent[7] = "2.openEulerVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + } + if i.AbiVersion != "" { + versionAbiFlag := true + abiVersionArry := strings.Split(i.AbiVersion, ",") + if len(abiVersionArry) > 0 { + for _, affectx := range abiVersionArry { + affect := common.BranchVersionRep(affectx) + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "是" || versionArry[1] == "否" { + continue + } else { + versionAbiFlag = false + break + } + } else { + versionAbiFlag = false + break + } + } + } + if !versionAbiFlag { + tbContent[12] = "待分析" + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = fmt.Sprintf("4.修复是否涉及abi变化(是/否)=> 没有分析或未按正确格式填写:%v", i.AbiVersion) + msg = fmt.Sprintf("4.修复是否涉及abi变化(是/否)=> 没有分析或未按正确格式填写:%v", i.AbiVersion) + ok = false + } else { + tbContent[12] = "已分析" + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = util.TrimStringNR(i.AbiVersion) + } + } else { + tbContent[12] = "已分析" + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = "" + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkGaussIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { + tbContent[0] = "待分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + } else { + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + } + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openGaussScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openGaussVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + tbContent[3] = "待分析" + tbContent[4] = "2.openGaussScore" + tbContent[5] = fmt.Sprintf("2.openGaussScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openGaussScore=> 没有填写或正确填写(0-10)") + ok = false + } else { + tbContent[3] = "已分析" + tbContent[4] = "2.openGaussScore" + tbContent[5] = i.OpenEulerScore + } + if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { + tbContent[6] = "待分析" + tbContent[7] = "2.openGaussVector" + tbContent[8] = fmt.Sprintf("2.openGaussVector=> 没有正确填写") + msg = fmt.Sprintf("2.openGaussVector=> 没有正确填写") + ok = false + } else { + tbContent[6] = "已分析" + tbContent[7] = "2.openGaussVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkSporeIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { + tbContent[0] = "待分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + } else { + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + } + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + tbContent[3] = "待分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = fmt.Sprintf("2.MindSporeScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.MindSporeScore=> 没有填写或正确填写(0-10)") + ok = false + } else { + tbContent[3] = "已分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = i.OpenEulerScore + } + if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { + tbContent[6] = "待分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = fmt.Sprintf("2.MindSporeVector=> 没有正确填写") + msg = fmt.Sprintf("2.MindSporeVector=> 没有正确填写") + ok = false + } else { + tbContent[6] = "已分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkLooKengIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { + tbContent[0] = "待分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + } else { + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + } + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + tbContent[3] = "待分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = fmt.Sprintf("2.openLooKengScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openLooKengScore=> 没有填写或正确填写(0-10)") + ok = false + } else { + tbContent[3] = "已分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = i.OpenEulerScore + } + if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { + tbContent[6] = "待分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = fmt.Sprintf("2.openLooKengVector=> 没有正确填写") + msg = fmt.Sprintf("2.openLooKengVector=> 没有正确填写") + ok = false + } else { + tbContent[6] = "已分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func commentUpdateIssue(issueTmp models.IssueTemplate, owner, accessToken, path string) { + if accessToken != "" && owner != "" && path != "" { + cvlnCenter := models.VulnCenter{} + err := models.GetVulnCenterByCVEID(&cvlnCenter, issueTmp.CveId) + if err != nil { + logs.Error("GetVulnCenterByCVEID, err: ", err, ",cvlnCenter: ", cvlnCenter) + } else { + _, err := taskhandler.UpdateIssueToGit(accessToken, owner, path, cvlnCenter, issueTmp) + if err != nil { + logs.Error("UpdateIssueToGit, upErr: ", err, ",issueTmp: ", issueTmp) + } + } + } +} + +func saveVectorData(vct string, cveID int64) error { + score, err := models.QueryIssueScore(cveID) + if err != nil { + return err + } + if vct == "" { + return errors.New("vct value is empty") + } + upFields := make([]string, 0) + score.OvectorVule = vct + upFields = append(upFields, "o_vector_value") + vMap, ok := util.VctToMap(vct) + if !ok { + return errors.New("vector value illegal") + } + if util.RegexpVectorV2.Match([]byte(vct)) { + //update v2 vector + avv := util.ReadVMValueV2(vMap["AV"]) + if avv != "" { + score.OaccessVector = avv + upFields = append(upFields, "o_access_vector") + } + acv := util.ReadVMValueV2(vMap["AC"]) + if acv != "" { + score.OaccessComplexity = acv + upFields = append(upFields, "o_access_complexity") + } + au := util.ReadVMValueV2(vMap["Au"]) + if au != "" { + score.Oauthentication = au + upFields = append(upFields, "o_authentication") + } + cv := util.ReadVMValueV2(vMap["C"]) + if cv != "" { + score.Oconfidentiality = cv + upFields = append(upFields, "o_confidentiality") + } + iv := util.ReadVMValueV2(vMap["I"]) + if iv != "" { + score.Ointegrity = iv + upFields = append(upFields, "o_integrity") + } + av := util.ReadVMValueV2(vMap["A"]) + if av != "" { + score.Oavailability = av + upFields = append(upFields, "o_availability") + } + } else { + //update v3 vector + avv := util.ReadVMValue(vMap["AV"]) + if avv != "" { + score.OattackVector = avv + upFields = append(upFields, "o_attack_vector") + } + acv := util.ReadVMValue(vMap["AC"]) + if acv != "" { + score.OattackComplexity = acv + upFields = append(upFields, "o_attack_complexity") + } + prv := util.ReadVMValue(vMap["PR"]) + if prv != "" { + score.OprivilegeRequired = prv + upFields = append(upFields, "o_privilege_required") + } + uiv := util.ReadVMValue(vMap["UI"]) + if uiv != "" { + score.OuserInteraction = uiv + upFields = append(upFields, "o_user_interaction") + } + sv := util.ReadVMValue(vMap["S"]) + if sv != "" { + score.Oscope = sv + upFields = append(upFields, "o_scope") + } + cv := util.ReadVMValue(vMap["C"]) + if cv != "" { + score.Oconfidentiality = cv + upFields = append(upFields, "o_confidentiality") + } + iv := util.ReadVMValue(vMap["I"]) + if iv != "" { + score.Ointegrity = iv + upFields = append(upFields, "o_integrity") + } + av := util.ReadVMValue(vMap["A"]) + if av != "" { + score.Oavailability = av + upFields = append(upFields, "o_availability") + } + } + if len(upFields) > 0 { + //Perform update + err = models.UpdateScore(&score, upFields...) + if err != nil { + return err + } + } + return nil +} + +func handleCommentPackage(packageStr string, cveID int64) error { + packageStr = util.TrimString(packageStr) + err := models.UpdatePackageByCveId(packageStr, cveID) + if err != nil { + return err + } + return nil +} + +//CloseIssue close gitee issue +func CloseIssue(token, repo, issueNum, owner string) bool { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/issues/%s", owner, issueNum) + param := struct { + AccessToken string `json:"access_token"` + Repo string `json:"repo"` + State string `json:"state"` + }{token, repo, "closed"} + pj, err := json.Marshal(¶m) + if err != nil { + logs.Error("json.Marshal, err:", err, ",issueNum: ", issueNum) + return false + } + return UpdateGiteIssue(url, pj) +} + +//UpdateGiteIssue update gitee issue +func UpdateGiteIssue(url string, param []byte) bool { + read := bytes.NewReader(param) + req, err := http.NewRequest(http.MethodPatch, url, read) + if err != nil { + logs.Error("NewRequest, err:", err) + return false + } + defer req.Body.Close() + req.Header.Set("Content-Type", "application/json;charset=UTF-8") + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("client.Do, err:", err) + return false + } + defer resp.Body.Close() + if resp.StatusCode == 200 { + return true + } + return false +} + +func AddGitIssue(issueHook *models.IssuePayload, desc, product string) error { + issueTitle := util.TrimString(issueHook.Title) + issueType := util.TrimString(issueHook.Issue.TypeName) + issueNumber := util.TrimString(issueHook.Issue.Number) + issueState := util.TrimString(issueHook.Issue.State) + issueZhState := util.TrimString(issueHook.Issue.StateName) + repoPath := util.TrimString(issueHook.Repository.Path) + nameSpace := util.TrimString(issueHook.Repository.NameSpace) + organizationID := int8(1) + organizationID = taskhandler.GetOrganizationId(nameSpace) + issueTitle = taskhandler.RegMatchCve(issueTitle) + if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + item := models.GiteOriginIssue{IssueId: issueHook.Issue.Id, Url: issueHook.Issue.HtmlUrl, + Number: issueNumber, State: issueState, Title: issueTitle, + IssueType: issueType, SecurityHole: true, + IssueCreateAt: issueHook.Issue.CreateAt, IssueUpdateAt: issueHook.Issue.UpdateAt, + IssueFinishAt: issueHook.Issue.FinishedAt, IssueCreate: issueHook.Issue.User.Login, + IssueAssignee: issueHook.Assignee.Login, RepoPath: repoPath, + RepoUrl: issueHook.Repository.Url, InfProduct: product, + RepoDesc: desc, IssueState: issueZhState, Owner: nameSpace, OrganizationID: organizationID, + Status: 0} + //vt := util.TrimString(v.Title) + cveStr := "" + if strings.HasPrefix(issueTitle, "CVE") { + cveStr = issueTitle + } else if issueHook.Issue.Body != "" { + // Use regular expressions to intercept the body, which will be improved later + sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + cveStr = sm[0][1] + } + } + cveList := []string{} + if len(cveStr) > 1 { + cveRes := util.RegexpCveNumVaule.FindAllStringSubmatch(cveStr, -1) + if len(cveRes) > 0 { + for _, cr := range cveRes { + if len(cr) > 0 { + tCr := util.TrimString(cr[0]) + if _, ok := common.FindSliceEm(cveList, tCr); !ok { + cveList = append(cveList, tCr) + } + } + } + } + } + owner, accessToken := common.GetOwnerAndToken(organizationID) + for _, cve := range cveList { + item.CveNumber = cve + // Check whether the current cve has created an issue + checkRes := taskhandler.OPenCheckWhetherIssue(item.CveNumber, repoPath, owner, accessToken, organizationID) + if checkRes { + logs.Error("Cve has created an issue, please process the previous issue first, ", + item.CveNumber, repoPath, ", organizationID: ", organizationID) + taskhandler.AddCommentToIssue(fmt.Sprintf(HasCreateIssue, issueHook.Assignee.UserName, item.CveNumber), + issueNumber, owner, repoPath, accessToken) + return errors.New("Ignore the current issue") + } + //vb := util.TrimString(v.Body) + vb := strings.ReplaceAll(issueHook.Issue.Body, "：", ":") + item.Body = vb + item.IssueExistTpl = false + err := item.InsertOrUpdate(2) + if err != nil { + logs.Error("insert or update issue fail:", err) + } + } + return nil + } else { + return errors.New("Ignore the current issue") + } +} + +func DelOrgIssue(issueHook *models.IssuePayload, organizationID int8) { + issueTitle := util.TrimString(issueHook.Title) + issueType := util.TrimString(issueHook.Issue.TypeName) + issueNumber := util.TrimString(issueHook.Issue.Number) + issueState := util.TrimString(issueHook.Issue.State) + issueZhState := util.TrimString(issueHook.Issue.StateName) + repoPath := util.TrimString(issueHook.Repository.Path) + nameSpace := util.TrimString(issueHook.Repository.NameSpace) + organizationID = taskhandler.GetOrganizationId(nameSpace) + issueTitle = taskhandler.RegMatchCve(issueTitle) + if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + // Data deletion record + idr := models.IssueDeleteRecord{IssueId: issueHook.Issue.Id, IssueNum: issueNumber, + DelAssignee: issueHook.Sender.Login, Owner: nameSpace, Repo: repoPath, + DeleteTime: common.GetCurTime(), OrganizationID: organizationID, CveNum: issueTitle} + models.InsertIssueDeleteRecord(&idr) + item := models.GiteOriginIssue{IssueId: issueHook.Issue.Id, Url: issueHook.Issue.HtmlUrl, + Number: issueNumber, State: issueState, Title: issueTitle, + IssueType: issueType, SecurityHole: true, + IssueCreateAt: issueHook.Issue.CreateAt, IssueUpdateAt: issueHook.Issue.UpdateAt, + IssueFinishAt: issueHook.Issue.FinishedAt, IssueCreate: issueHook.Issue.User.Login, + IssueAssignee: "", RepoPath: repoPath, + RepoUrl: issueHook.Repository.Url, InfProduct: "", + RepoDesc: "", IssueState: issueZhState, Owner: nameSpace, OrganizationID: organizationID} + if strings.HasPrefix(issueTitle, "CVE") { + item.CveNumber = issueTitle + } else if issueHook.Issue.Body != "" { + // Use regular expressions to intercept the body and improve it later + sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + item.CveNumber = util.TrimString(sm[0][1]) + } + } + err := item.Detlete() + if err != nil { + logs.Error("Failed to delete issue:", err) + } + issueTmp := models.IssueTemplate{} + issueTmp.IssueId = issueHook.Issue.Id + issueTmp.IssueNum = issueNumber + issueTmp.Repo = repoPath + issueErr := models.GetIssueTemplateByColName(&issueTmp, "IssueNum", "Repo", "IssueId") + if issueErr != nil { + return + } + issueTmp.Status = 6 + issueTmp.IssueStatus = 6 + issueTmp.StatusName = "已删除" + tpErr := models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", + "status_name") + if tpErr != nil { + logs.Error("UpdateIssueTemplate, tpErr:", tpErr, ",issueTmp: ", issueTmp) + } + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum, PackName: issueTmp.Repo} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num", "pack_name") + if cveErr != nil { + return + } + cveCenter.Status = 7 + update := models.UpdateVulnCenter(&cveCenter, "cve_status") + if !update { + logs.Error("update vulnCenter fail ") + } + } +} + +// Entry function for handling issue status +func gitAddIssueProc(issueHook *models.IssuePayload, organizationID int8) error { + owner, token := common.GetOwnerAndToken(organizationID) + path := issueHook.Repository.Path + // The amount of data processed at a time + prcNum, err := beego.AppConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("crontab::prcnum, error: ", err) + return errors.New("value is nil") + } + // How many days have been processed to date data + days, ok := beego.AppConfig.Int("crontab::days") + if ok != nil { + logs.Error("crontab::days, err:", err) + return ok + } + // openeuler Number start value + cveRef := beego.AppConfig.String("cve::cveref") + openeulerNum, ok := beego.AppConfig.Int("cve::openeulernum") + if ok != nil { + logs.Error("cve::openeulernum, err:", err) + return ok + } + // Determine whether the issue has been created + product, err := taskhandler.GetInfProduct(token, owner, path) + if err != nil { + logs.Error("GetInfProduct, err: ", err) + } + desc := taskhandler.GetRepoDescription(path) + giErr := AddGitIssue(issueHook, desc, product) + if giErr != nil { + logs.Error("AddGitIssue, giErr: ", giErr) + } + // Compatible with created issue data + oki, err := taskhandler.GetCveIssueData(prcNum, days, openeulerNum, cveRef, owner, 1) + if !oki { + logs.Error("ProcCveOriginData, GetCveIssueData, err: ", err) + } + cError := task.CreateIssue() + if cError != nil { + logs.Error("CreateIssue, cError: ", cError) + } + comErr := AddIssueComment(token, owner, path, issueHook.Issue.Number, + issueHook.Issue.User.UserName, issueHook.Issue.Id) + logs.Info("CreateIssueToGit, Issue comment creation result, err: ", comErr) + return comErr +} + +func AddIssueComment(token, owner, path, issueNum, assignee string, issueId int64) error { + // Create the first comment + branchList := []string{} + errBrands := errors.New("") + issueTmp := models.IssueTemplate{} + issueTmp.IssueId = issueId + issueTmp.IssueNum = issueNum + issueTmp.Repo = path + issueErr := models.GetIssueTemplateByColName(&issueTmp, "IssueNum", "Repo", "IssueId") + if issueErr != nil { + logs.Error("GetIssueTemplateByColName, err: ", issueErr, ", issue: ", issueTmp) + return issueErr + } + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum, PackName: path} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num", "pack_name") + if cveErr != nil { + return cveErr + } + owner, token = common.GetOwnerAndToken(cveCenter.OrganizationID) + if cveCenter.OrganizationID == 4 { + cveList := strings.Split(cveCenter.CveVersion, ",") + if len(cveList) > 0 { + for _, vl := range cveList { + olky := models.OpenLookengYaml{PackageName: cveCenter.PackName, Version: vl} + looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") + if olky.Id > 0 { + path = olky.Repo + break + } + logs.Info("GetOpenLookengYaml, looKengErr: ", looKengErr) + } + } + branchList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("OpenLookeng GetBranchesInfo, Failed to obtain the branch information of the repo, ", path, ", err: ", errBrands) + return errors.New("Failed to obtain branch information") + } + } else if cveCenter.OrganizationID == 3 { + // Query the repo that needs to submit an issue + cveList := strings.Split(cveCenter.CveVersion, ",") + if len(cveList) > 0 { + for _, vl := range cveList { + ms := models.MindSporeYaml{PackageName: cveCenter.PackName, Version: vl} + sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") + if ms.Id > 0 { + path = ms.Repo + break + } + logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) + } + } + // Get branch information + branchList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("mindspore GetBranchesInfo, Failed to obtain the branch information of the repo, ", path, ", err: ", errBrands) + return errors.New("Failed to obtain branch information") + } + } else if cveCenter.OrganizationID == 2 { + path = beego.AppConfig.String("opengauss::openGauss-server") + // Get branch information + branchList, errBrands = taskhandler.GetBranchesInfo(token, owner, path, cveCenter.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("Gauss GetBranchesInfo, Failed to obtain the branch information of the repo, ", path, ", err: ", errBrands) + return errors.New("Failed to obtain branch information") + } + } else { + // Get branch information + branchList, errBrands = taskhandler.GetBranchesInfo(token, owner, path, cveCenter.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("Euler GetBranchesInfo, Failed to obtain the branch information of the repo, ", path, ", err: ", errBrands) + return errors.New("Failed to obtain branch information") + } + } + // Create issue comment + affectedVersion := "" + if len(branchList) > 0 { + for i, brand := range branchList { + if brand == "" || len(brand) < 2 { + continue + } + brandx := "" + if cveCenter.OrganizationID == 1 { + brandx = common.BranchVersionRep(brand) + brandx = taskhandler.OrgRepoParams(cveCenter.PackName, brandx) + } else { + brandx = brand + } + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brandx + ":\n" + } + } else { + affectedVersion = affectedVersion + "\n" + } + _, assignLoginList, _ := taskhandler.GetRepoMember(token, owner, path) + errx := taskhandler.CreateIssueComment(token, owner, path, assignee, + cveCenter, issueNum, affectedVersion, assignLoginList) + return errx +} + +func gitDelIssueProc(issueHook *models.IssuePayload, organizationID int8) error { + DelOrgIssue(issueHook, organizationID) + return nil +} diff --git a/cve-vulner-manager/controllers/issue.go b/cve-vulner-manager/controllers/issue.go new file mode 100644 index 0000000000000000000000000000000000000000..abb34a7657070d00b170a60f54f7fc2ffea85143 --- /dev/null +++ b/cve-vulner-manager/controllers/issue.go @@ -0,0 +1,108 @@ +package controllers + +import ( + "cvevulner/errcode" + "cvevulner/models" + "encoding/json" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" +) + +type IssueOathCallbackController struct { + beego.Controller +} + +func (c *IssueOathCallbackController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type IssueRepoWhiteData struct { + RepoId int64 `json:"id"` + PackageName string `json:"packageName"` + Version string `json:"version"` + Branchs string `json:"branchs"` + Status int8 `json:"status"` + CreateTime string `json:"createTime"` + UpdateTime string `json:"updateTime"` + DeleteTime string `json:"deleteTime"` +} + + +// @Title UserLogin +// @Description UserLogin +// @Param body body models.User true "body for user content" +// @Success 200 {int} models.User.ID +// @Failure 403 body is empty +// @router / [post] +func (c *IssueOathCallbackController) Post() { + req := make(map[string]interface{}) + resp := make(map[string]interface{}) + resp["errno"] = errcode.RecodeLoginErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeLoginErr) + resp["body"] = Result{} + defer c.RetData(resp) + json.Unmarshal(c.Ctx.Input.RequestBody, &req) + logs.Info("Login request parameters: ", &req) + return +} + +type CveIssueWhiteListController struct { + beego.Controller +} + +func (c *CveIssueWhiteListController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title Get issuewhitelist +// @Description get packages +// @Param status int true (0,1,2) +// @Success 200 {object} models.IssueRepoWhitelist +// @Failure 403 :status is err +// @router / [get] +func (u *CveIssueWhiteListController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var ird []IssueRepoWhiteData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = []IssueRepoWhiteData{} + //defer u.RetData(resp) + status, err := u.GetInt8("status", 0) + if err != nil { + logs.Error("status, err: ", err, ", status: ", status) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + issueWhiteData, issueErr := models.QueryIssueWhitelist(status) + if issueErr == nil && len(issueWhiteData) > 0 { + for _, issues := range issueWhiteData { + var irda IssueRepoWhiteData + irda.CreateTime = issues.CreateTime + irda.Status = issues.Status + irda.Version = issues.Version + irda.PackageName = issues.PackageName + irda.UpdateTime = issues.UpdateTime + irda.RepoId = issues.RepoId + irda.DeleteTime = issues.DeleteTime + irda.Branchs = issues.Branchs + ird = append(ird, irda) + } + resp["body"] = ird + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetData(resp) + } else { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } +} diff --git a/cve-vulner-manager/controllers/login.go b/cve-vulner-manager/controllers/login.go new file mode 100644 index 0000000000000000000000000000000000000000..3b91a92c2727b12c574c8541367224df7f255ab3 --- /dev/null +++ b/cve-vulner-manager/controllers/login.go @@ -0,0 +1,90 @@ +package controllers + +import ( + "cvevulner/common" + "cvevulner/errcode" + "cvevulner/models" + "encoding/json" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "time" +) + +type UserLoginController struct { + beego.Controller +} + +type Result struct { + Key string `json:"Token"` + UserId interface{} `json:"UserId"` +} + +func (c *UserLoginController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +// @Title UserLogin +// @Description UserLogin +// @Param body body models.User true "body for user content" +// @Success 200 {int} models.User.ID +// @Failure 403 body is empty +// @router / [post] +func (u *UserLoginController) Post() { + req := make(map[string]interface{}) + resp := make(map[string]interface{}) + resp["errno"] = errcode.RecodeLoginErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeLoginErr) + resp["body"] = Result{} + //defer u.RetData(resp) + err := json.Unmarshal(u.Ctx.Input.RequestBody, &req) + if err != nil { + logs.Error(err) + u.RetData(resp) + return + } + //Judge whether it is legal + if req["userName"] == nil || req["passWord"] == nil { + resp["errno"] = errcode.RecodeDataErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeDataErr) + resp["body"] = Result{} + logs.Error("Data error: username or password") + u.RetData(resp) + return + } + password := fmt.Sprintf("%s", req["passWord"]) + // Encryption first comment + password = common.DesString(password) + if password == "" || len(password) == 0 { + resp["errno"] = errcode.RecodePwdErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodePwdErr) + logs.Error("Password parsing error.") + resp["body"] = Result{} + u.RetData(resp) + return + } + var strc Result + username := fmt.Sprintf("%s", req["userName"]) + resp_model, err := models.GetCveUserByUser(username, password) + logs.Info(resp_model) + if resp_model != nil && err == nil { + token, terr := common.GenToken(username, password) + if terr == nil { + strc.Key = token + logs.Info(resp_model[0]["user_id"]) + user_id := resp_model[0]["user_id"] + strc.UserId = user_id + resp["body"] = strc + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + expirTime := common.GetTokenExpirTime() + newTime := time.Now().AddDate(0, 0, expirTime) + models.UpdateToken(resp_model[0]["user_id"], token, newTime) + u.RetData(resp) + return + } + } + u.RetData(resp) + return +} diff --git a/cve-vulner-manager/controllers/object.go b/cve-vulner-manager/controllers/object.go new file mode 100644 index 0000000000000000000000000000000000000000..bde796f65b5b48733e912f6948c640ac5f356737 --- /dev/null +++ b/cve-vulner-manager/controllers/object.go @@ -0,0 +1,97 @@ +package controllers + +import ( + "cvevulner/models" + "encoding/json" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" +) + +//ObjectController Operations about object +type ObjectController struct { + beego.Controller +} + +// @Title Create +// @Description create object +// @Param body body models.Object true "The object content" +// @Success 200 {string} models.Object.ID +// @Failure 403 body is empty +// @router / [post] +func (o *ObjectController) Post() { + var ob models.Object + err := json.Unmarshal(o.Ctx.Input.RequestBody, &ob) + if err != nil { + logs.Error(err) + o.Abort("500") + } + objectId := models.AddOne(ob) + o.Data["json"] = map[string]string{"ObjectId": objectId} + o.ServeJSON() +} + +// @Title Get +// @Description find object by objectid +// @Param objectId path string true "the objectid you want to get" +// @Success 200 {object} models.Object +// @Failure 403 :objectId is empty +// @router /:objectId [get] +func (o *ObjectController) Get() { + objectId := o.Ctx.Input.Param(":objectId") + if objectId != "" { + ob, err := models.GetOne(objectId) + if err != nil { + o.Data["json"] = err.Error() + } else { + o.Data["json"] = ob + } + } + o.ServeJSON() +} + +// @Title GetAll +// @Description get all objects +// @Success 200 {object} models.Object +// @Failure 403 :objectId is empty +// @router / [get] +func (o *ObjectController) GetAll() { + obs := models.GetAll() + o.Data["json"] = obs + o.ServeJSON() +} + +// @Title Update +// @Description update the object +// @Param objectId path string true "The objectid you want to update" +// @Param body body models.Object true "The body" +// @Success 200 {object} models.Object +// @Failure 403 :objectId is empty +// @router /:objectId [put] +func (o *ObjectController) Put() { + objectId := o.Ctx.Input.Param(":objectId") + var ob models.Object + err := json.Unmarshal(o.Ctx.Input.RequestBody, &ob) + if err != nil { + o.Data["json"] = err.Error() + } + err = models.Update(objectId, ob.Score) + if err != nil { + o.Data["json"] = err.Error() + } else { + o.Data["json"] = "update success!" + } + o.ServeJSON() +} + +// @Title Delete +// @Description delete the object +// @Param objectId path string true "The objectId you want to delete" +// @Success 200 {string} delete success! +// @Failure 403 objectId is empty +// @router /:objectId [delete] +func (o *ObjectController) Delete() { + objectId := o.Ctx.Input.Param(":objectId") + models.Delete(objectId) + o.Data["json"] = "delete success!" + o.ServeJSON() +} diff --git a/cve-vulner-manager/controllers/packages.go b/cve-vulner-manager/controllers/packages.go new file mode 100644 index 0000000000000000000000000000000000000000..c712e371566dc0a2ffeb0de9333653808f430609 --- /dev/null +++ b/cve-vulner-manager/controllers/packages.go @@ -0,0 +1,317 @@ +package controllers + +import ( + "cvevulner/errcode" + "cvevulner/models" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "strings" +) + +// Operations about Packages +type PackagesController struct { + beego.Controller +} + +func (c *PackagesController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type PackagesInfoController struct { + beego.Controller +} + +func (c *PackagesInfoController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type PackageData struct { + Id int64 `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Release string `json:"release"` + OriginUrl string `json:"url"` + CpeName string `json:"cpeName"` + ReleaseTime string `json:"releaseTime"` + LatestVersion string `json:"latestVersion"` + LatestVersionTime string `json:"latestVersionTime"` +} + +type Provide struct { + Name string `json:"name"` + Requiredby []string `json:"requiredby"` +} + +type Require struct { + Name string `json:"name"` + Providedby []string `json:"providedby"` +} + +type SubPack struct { + Name string `json:"name"` + Provides []Provide `json:"provides"` + Requires []Require `json:"requires"` +} + +type PackageInfoData struct { + PkgName string `json:"pkgName"` + Version string `json:"version"` + Release string `json:"release"` + OriginUrl string `json:"url"` + GiteeUrl string `json:"giteeUrl"` + Summary string `json:"summary"` + Description string `json:"description"` + BuildRequired []string `json:"buildRequired"` + Subpack []SubPack `json:"subpack"` +} + +// @Title Get packages +// @Description get packages +// @Param pageNum pageSize int true +// @Success 200 {object} models.package +// @Failure 403 :pageNum is err +// @router / [get] +func (u *PackagesController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var pd []PackageData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = []PackageData{} + resp["totalCount"] = 0 + resp["totalPage"] = 0 + //defer u.RetData(resp) + var iw models.IpWhite + if addr != "" { + addrIp := strings.Split(addr, ":") + err := models.GetIpWhite(addrIp[0], &iw) + if err != nil { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + } else { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + token := u.GetString("token") + if token == "" { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + u.RetData(resp) + return + } else { + ok := models.CheckToken(token) + if !ok { + resp["errno"] = errcode.RecodeRoleErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeRoleErr) + u.RetData(resp) + return + } + } + PageNum, err := u.GetInt64("pageNum") + if err != nil { + logs.Error("pageNum, err: ", err) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + PageSize, err := u.GetInt64("pageSize") + if err != nil { + logs.Error("PageSize, err: ", err) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + QueryPkgName := u.GetString("queryPkgName") + if QueryPkgName != "" { + logs.Info("Query data: ", QueryPkgName, ", Information about the package") + } + + totalNum := models.GetPackageNum(QueryPkgName) + if totalNum > 0 { + if PageSize >= totalNum { + resp["totalPage"] = 1 + } else { + if totalNum%PageSize == 0 { + resp["totalPage"] = totalNum / PageSize + } else { + totalPage := totalNum / PageSize + totalPage += 1 + resp["totalPage"] = totalPage + } + } + resp["totalCount"] = totalNum + ge, num, err := models.GetPackageList(PageSize, PageNum, QueryPkgName) + if num > 0 && err == nil { + for _, g := range ge { + var pg PackageData + pg.Id = g.GitId + pg.Release = g.Release + pg.Version = g.Version + pg.CpeName = g.CpePackName + pg.LatestVersion = g.LatestVersion + pg.LatestVersionTime = g.LatestVersionTime + pg.Name = g.PackageName + pg.OriginUrl = g.OriginUrl + pg.ReleaseTime = g.ReleaseTime + pd = append(pd, pg) + } + resp["body"] = pd + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetData(resp) + } + } else { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } +} + +// @Title Get packagesinfo +// @Description get packagesinfo +// @Param pkgName token string true +// @Success 200 {object} models.package +// @Failure 403 :pkgName is err +// @router / [get] +func (u *PackagesInfoController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ", Header: ", req.Header, ", body: ", req.Body) + resp := make(map[string]interface{}) + var pd PackageInfoData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = PackageInfoData{} + //defer u.RetData(resp) + var iw models.IpWhite + if addr != "" { + addrIp := strings.Split(addr, ":") + err := models.GetIpWhite(addrIp[0], &iw) + if err != nil { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + } else { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + token := u.GetString("token") + if token == "" { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + u.RetData(resp) + return + } else { + ok := models.CheckToken(token) + if !ok { + resp["errno"] = errcode.RecodeRoleErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeRoleErr) + u.RetData(resp) + return + } + } + pkgName := u.GetString("pkgName") + if pkgName == "" { + logs.Error("pkgName, Parameter error") + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + version := u.GetString("version") + if version == "" { + logs.Error("version, Parameter error") + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + u.RetData(resp) + return + } + var gi models.GitPackageInfo + err := models.GetPackageInfo(pkgName, version, &gi) + if err != nil { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } + pd.OriginUrl = gi.OriginUrl + pd.Version = gi.Version + pd.PkgName = gi.PackageName + pd.Release = gi.Release + pd.Description = gi.Decription + pd.GiteeUrl = gi.GitUrl + pd.Summary = gi.Summary + if gi.BuildRequired != "" { + buildRequired := strings.Split(gi.BuildRequired, ",") + pd.BuildRequired = buildRequired + } else { + pd.BuildRequired = []string{} + } + gs, num, err := models.GetSubPackage(gi.DetailId) + if num > 0 && err == nil { + for _, gg := range gs { + var sp SubPack + sp.Name = gg.SubPackName + gsp, numx, errx := models.GetSubPackProvide(gg.SubId) + if numx > 0 && errx == nil { + for _, gps := range gsp { + var ssp Provide + ssp.Name = gps.ProvideName + gspr, numxx, errxx := models.GetSubPackrequiredby(gps.ProvideId) + if numxx > 0 && errxx == nil { + for _, grq := range gspr { + ssp.Requiredby = append(ssp.Requiredby, grq.Requiredby) + } + } else { + ssp.Requiredby = []string{} + } + sp.Provides = append(sp.Provides, ssp) + } + } else { + sp.Provides = []Provide{} + } + gpre, numm, err := models.GetSubPackRequire(gg.SubId) + if numm > 0 && err == nil { + for _, ges := range gpre { + var rssp Require + rssp.Name = ges.RequireName + if ges.Providedby != "" { + rssp.Providedby = strings.Split(ges.Providedby, ",") + } else { + rssp.Providedby = []string{} + } + sp.Requires = append(sp.Requires, rssp) + } + } else { + sp.Requires = []Require{} + } + pd.Subpack = append(pd.Subpack, sp) + } + + } else { + pd.Subpack = []SubPack{} + } + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + resp["body"] = pd + u.RetData(resp) + return +} diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go new file mode 100644 index 0000000000000000000000000000000000000000..48e9facd7617588302a670fda5548a5892d04dfd --- /dev/null +++ b/cve-vulner-manager/controllers/upload.go @@ -0,0 +1,441 @@ +package controllers + +import ( + "cvevulner/common" + "cvevulner/errcode" + "cvevulner/models" + "cvevulner/task" + "encoding/json" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "strings" +) + +type UserUploadController struct { + beego.Controller +} + +type ResultData struct { + CveNum string `json:"CveNum"` + Status int `json:"Status"` +} + +func (c *UserUploadController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() + // sysnc cve and create issue + synErr := task.SyncCveAndIssue() + if synErr != nil { + logs.Error("SyncCveAndIssue, Sync cve data error, err: ", synErr) + } else { + logs.Info("SyncCveAndIssue, cve data has been synchronized") + } +} + +func (c *CveErrorFeedBackController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + +type CveErrorFeedBackController struct { + beego.Controller +} + +type CveErrorData struct { + CveNum string `json:"cveNum"` + ErrorDesc string `json:"errorDesc"` + CveUploadTime string `json:"cveUploadTime"` +} + +// @Title Get cveerrorfeedback +// @Description get cveerrorfeedback +// @Param startDate endDate string false +// @Success 200 {object} models.uploadcve +// @Failure 403 :endDate is err +// @router / [get] +func (u *CveErrorFeedBackController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, + ",Header: ", req.Header, ",body: ", req.Body) + resp := make(map[string]interface{}) + var ced []CveErrorData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = []CveErrorData{} + //defer u.RetData(resp) + var iw models.IpWhite + if addr != "" { + addrIp := strings.Split(addr, ":") + err := models.GetIpWhite(addrIp[0], &iw) + if err != nil { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + } else { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + token := u.GetString("token") + if token == "" { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + u.RetData(resp) + return + } else { + ok := models.CheckToken(token) + if !ok { + resp["errno"] = errcode.RecodeRoleErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeRoleErr) + u.RetData(resp) + return + } + } + startDate := u.GetString("startDate") + if startDate != "" { + logs.Info("Query start date: ", startDate) + } + + endDate := u.GetString("endDate") + if endDate != "" { + logs.Info("Query end date: ", endDate) + } else { + endDate = common.GetCurTime() + } + + it, ok := models.QueryCveErrorInfo(3, startDate, endDate) + if ok { + for _, cveErr := range it { + var ved CveErrorData + ved.CveNum = cveErr.CveNum + ved.CveUploadTime = common.TimeConverStr(cveErr.CreateTime.String()[:19]) + ved.ErrorDesc = cveErr.ErrorDescription + ced = append(ced, ved) + } + resp["body"] = ced + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + u.RetData(resp) + } else { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + u.RetData(resp) + return + } +} + +// @Title UserUpload +// @Description UserUpload +// @Param body body models.OriginUpstream true "body for user content" +// @Success 200 {int} models.OriginUpstream.CveId +// @Failure 403 body is empty +// @router / [post] +func (u *UserUploadController) Post() { + var uploaddata common.UploadData + var ResDataList []ResultData + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "Client request ip address: ", addr, ",Header: ", req.Header) + resp := make(map[string]interface{}) + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = []ResultData{} + //defer u.RetData(resp) + json.Unmarshal(u.Ctx.Input.RequestBody, &uploaddata) + logs.Info("Cve upload request parameters: ", string(u.Ctx.Input.RequestBody)) + var iw models.IpWhite + if addr != "" { + addrIp := strings.Split(addr, ":") + err := models.GetIpWhite(addrIp[0], &iw) + if err != nil { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + } else { + resp["errno"] = errcode.RecodeIpErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) + u.RetData(resp) + return + } + //Judge whether it is legal + if uploaddata.Token == "" { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + resp["body"] = []ResultData{} + logs.Error("token request parameter is empty!") + u.RetData(resp) + return + } else { + // Check token + ok := models.CheckToken(uploaddata.Token) + if !ok { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + resp["body"] = []ResultData{} + logs.Error("token verification failed!") + u.RetData(resp) + return + } + } + if uploaddata.CveData == nil || len(uploaddata.CveData) == 0 { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + resp["body"] = []ResultData{} + logs.Error("cve data is empty") + u.RetData(resp) + return + } + for _, CveDataDict := range uploaddata.CveData { + resDataList := AddOrgUpstream(CveDataDict) + if len(resDataList) > 0 { + ResDataList = append(ResDataList, resDataList...) + } + } + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + resp["body"] = ResDataList + u.RetData(resp) + return +} + +func AddOrgUpstream(CveDataDict common.CveOriginData) (ResDataList []ResultData) { + defer common.Catchs() + logs.Info("Each request parameter: ", CveDataDict) + // Record data flow + AddOrgUpstreamRecord(CveDataDict) + var ResData ResultData + ids := CveDataDict.Ids + if len(ids) < 1 { + ResData.CveNum = ids + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + logs.Error("ids is null, cveDataDict:", CveDataDict) + return + } + cveNum := CveDataDict.CveNum + if len(cveNum) < 2 { + ResData.CveNum = cveNum + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + logs.Error("CveNum is null, cveDataDict:", CveDataDict) + return + } + updateType := CveDataDict.UpdateType + cvePackName := "" + if len(CveDataDict.CvePackName) > 0 { + cvePackName = strings.Join(CveDataDict.CvePackName, ",") + } + packName := "" + if len(CveDataDict.PackName) > 0 { + packName = strings.Join(CveDataDict.PackName, ",") + } else { + ResData.CveNum = cveNum + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + logs.Error("PackName is null, cveDataDict:", CveDataDict) + return + } + title := CveDataDict.Title + affectProduct := "" + if len(CveDataDict.AffectProduct) > 0 { + affectProduct = strings.Join(CveDataDict.AffectProduct, ",") + } else { + affectProduct = packName + } + cnnvdID := CveDataDict.CnnvdID + cnvdID := CveDataDict.CnvdID + publishedDate := CveDataDict.PublishedDate + vulStatus := CveDataDict.VulStatus + version := CveDataDict.Version + if version == "" { + version = packName + } + var orCve models.OriginUpstream + if ids != "" { + ids = common.DeletePreAndSufSpace(ids) + } + orCve.Ids = ids + if cveNum != "" { + cveNum = common.DeletePreAndSufSpace(cveNum) + } + orCve.CveNum = cveNum + orCve.Version = version + orCve.UpdateType = updateType + orCve.CvePackName = cvePackName + orCve.Credibility = CveDataDict.Credibility + if packName != "" { + packName = common.DeletePreAndSufSpace(packName) + } + orCve.PackName = packName + orCve.Title = title + if affectProduct == "" { + orCve.AffectProduct = packName + } + orCve.CnnvdID = cnnvdID + orCve.CnvdID = cnvdID + orCve.IsExit = 0 + orCve.PublishedDate = publishedDate + orCve.FirstPerTime = CveDataDict.GetTime + orCve.FirstGetTime = CveDataDict.EndGetTime + checkPackageAndVersion(packName, &orCve) + orCve.VulStatus = vulStatus + if strings.ToLower(updateType) == "delete" { + orCve.Status = 3 + } else if strings.ToLower(updateType) == "update" { + orCve.Status = 1 + } else { + orCve.Status = 0 + } + orCve.CreateTime = common.GetCurTime() + orCve.UpdateTime = common.GetCurTime() + var od models.OriginUpstreamDesc + od.EnDescription = CveDataDict.Description.EnDesc + od.ZhDescription = CveDataDict.Description.ZhDesc + var ous models.OriginUpstreamConfig + ous.Nodes = " " + var osi models.OriginUpstreamImpact + osi.Impact = " " + var osp models.OriginUpstreamPoc + osp.Url = CveDataDict.Poc.Url + osp.Date = CveDataDict.Poc.Date + osp.Dbindex = CveDataDict.Poc.Dbindex + osp.Desc = CveDataDict.Poc.Desc + osp.Path = CveDataDict.Poc.Path + osp.Source = CveDataDict.Poc.Source + var ose models.OriginUpstreamEvent + ose.Date = CveDataDict.Event.Date + ose.Url = CveDataDict.Event.Url + ose.Description = CveDataDict.Event.Description + ose.Title = CveDataDict.Event.Title + var osv models.OriginUpstreamVulType + osv.ZhDesc = CveDataDict.VulType.Zh + osv.EnDesc = CveDataDict.VulType.En + osv.Cwe = CveDataDict.VulType.Cwe + var osf models.OriginUpstreamFixSuggest + osf.Detail = CveDataDict.FixSuggest.Detail + dbCve, ok := models.QueryCveOriginByIds(ids) + if ok { + if orCve.Status != 3 { + orCve.Status = 1 + } + orCve.UpdateTime = common.GetCurTime() + if orCve.Status == 3 { + orCve.DeleteTime = common.GetCurTime() + } + logs.Info("The currently inserted data already exists: ", dbCve) + } + _, err := models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, &ose, &osv, &osf) + if err == nil { + logs.Info("Cve original data is successfully created CveNum: ", CveDataDict.Ids) + ResData.CveNum = CveDataDict.Ids + ResData.Status = 0 + ResDataList = append(ResDataList, ResData) + } else { + logs.Info("cve creation failed CveNum: ", CveDataDict.Ids) + ResData.CveNum = CveDataDict.Ids + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + } + return +} + +func checkPackageAndVersion(packName string, orCve *models.OriginUpstream) { + packNameList := []string{} + if packName != "" && len(packName) > 0 { + packNameList = strings.Split(packName, ",") + } + if len(packNameList) > 0 { + for _, pk := range packNameList { + pkList := strings.Split(pk, "==") + if len(pkList) == 2 { + var gits models.GitOpenEuler + gits.PackageName = pkList[0] + gits.Version = pkList[1] + gits.Status = 1 + ok := models.QueryCveOpeneulerdata(&gits) + if ok { + orCve.IsExit = 1 + logs.Info("The data corresponding to src-openEuler is: ", gits) + break + } else { + opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + orCve.IsExit = 1 + break + } + } + } + } + } +} + +func AddOrgUpstreamRecord(CveDataDict common.CveOriginData) { + orCve := models.OriginUpstreamRecord{} + ids := CveDataDict.Ids + cveNum := CveDataDict.CveNum + updateType := CveDataDict.UpdateType + cvePackName := "" + if len(CveDataDict.CvePackName) > 0 { + cvePackName = strings.Join(CveDataDict.CvePackName, ",") + } + packName := "" + if len(CveDataDict.PackName) > 0 { + packName = strings.Join(CveDataDict.PackName, ",") + } + title := CveDataDict.Title + affectProduct := "" + if len(CveDataDict.AffectProduct) > 0 { + affectProduct = strings.Join(CveDataDict.AffectProduct, ",") + } + cnnvdID := CveDataDict.CnnvdID + cnvdID := CveDataDict.CnvdID + publishedDate := CveDataDict.PublishedDate + vulStatus := CveDataDict.VulStatus + version := CveDataDict.Version + orCve.Ids = ids + if cveNum != "" { + cveNum = common.DeletePreAndSufSpace(cveNum) + } + orCve.CveNum = cveNum + orCve.Version = version + orCve.UpdateType = updateType + orCve.CvePackName = cvePackName + orCve.Credibility = CveDataDict.Credibility + if packName != "" { + packName = common.DeletePreAndSufSpace(packName) + } + orCve.PackName = packName + orCve.Title = title + if affectProduct == "" { + orCve.AffectProduct = packName + } + orCve.CnnvdID = cnnvdID + orCve.CnvdID = cnvdID + orCve.IsExit = 0 + orCve.PublishedDate = publishedDate + orCve.FirstPerTime = CveDataDict.GetTime + orCve.FirstGetTime = CveDataDict.EndGetTime + orCve.VulStatus = vulStatus + if strings.ToLower(updateType) == "delete" { + orCve.Status = 3 + } else if strings.ToLower(updateType) == "update" { + orCve.Status = 1 + } else { + orCve.Status = 0 + } + orCve.CreateTime = common.GetCurTime() + err := models.InsertOriginCveRecord(&orCve) + if err != nil { + logs.Error("InsertOriginCveRecord, err: ", err) + } + // Get the date one month ago + beforeDate := common.GetBeforeDate(1, -90) + models.DeleteOriginCveRecord(beforeDate) +} diff --git a/cve-vulner-manager/controllers/user.go b/cve-vulner-manager/controllers/user.go new file mode 100644 index 0000000000000000000000000000000000000000..3691759172db4ad7e2a34c78e726fddeac214a92 --- /dev/null +++ b/cve-vulner-manager/controllers/user.go @@ -0,0 +1,118 @@ +package controllers + +import ( + "cvevulner/models" + "encoding/json" + + "github.com/astaxie/beego" +) + +// Operations about Users +type UserController struct { + beego.Controller +} + +// @Title CreateUser +// @Description create users +// @Param body body models.User true "body for user content" +// @Success 200 {int} models.User.ID +// @Failure 403 body is empty +// @router / [post] +func (u *UserController) Post() { + var user models.User + json.Unmarshal(u.Ctx.Input.RequestBody, &user) + uid := models.AddUser(user) + u.Data["json"] = map[string]string{"uid": uid} + u.ServeJSON() +} + +// @Title GetAll +// @Description get all Users +// @Success 200 {object} models.User +// @router / [get] +func (u *UserController) GetAll() { + users := models.GetAllUsers() + u.Data["json"] = users + u.ServeJSON() +} + +// @Title Get +// @Description get user by uid +// @Param uid path string true "The key for staticblock" +// @Success 200 {object} models.User +// @Failure 403 :uid is empty +// @router /:uid [get] +func (u *UserController) Get() { + uid := u.GetString(":uid") + if uid != "" { + user, err := models.GetUser(uid) + if err != nil { + u.Data["json"] = err.Error() + } else { + u.Data["json"] = user + } + } + u.ServeJSON() +} + +// @Title Update +// @Description update the user +// @Param uid path string true "The uid you want to update" +// @Param body body models.User true "body for user content" +// @Success 200 {object} models.User +// @Failure 403 :uid is not int +// @router /:uid [put] +func (u *UserController) Put() { + uid := u.GetString(":uid") + if uid != "" { + var user models.User + json.Unmarshal(u.Ctx.Input.RequestBody, &user) + uu, err := models.UpdateUser(uid, &user) + if err != nil { + u.Data["json"] = err.Error() + } else { + u.Data["json"] = uu + } + } + u.ServeJSON() +} + +// @Title Delete +// @Description delete the user +// @Param uid path string true "The uid you want to delete" +// @Success 200 {string} delete success! +// @Failure 403 uid is empty +// @router /:uid [delete] +func (u *UserController) Delete() { + uid := u.GetString(":uid") + models.DeleteUser(uid) + u.Data["json"] = "delete success!" + u.ServeJSON() +} + +// @Title Login +// @Description Logs user into the system +// @Param username query string true "The username for login" +// @Param password query string true "The password for login" +// @Success 200 {string} login success +// @Failure 403 user not exist +// @router /login [get] +func (u *UserController) Login() { + username := u.GetString("username") + password := u.GetString("password") + if models.Login(username, password) { + u.Data["json"] = "login success" + } else { + u.Data["json"] = "user not exist" + } + u.ServeJSON() +} + +// @Title logout +// @Description Logs out current logged in user session +// @Success 200 {string} logout success +// @router /logout [get] +func (u *UserController) Logout() { + u.Data["json"] = "logout success" + u.ServeJSON() +} diff --git a/cve-vulner-manager/cve-py/Dockerfile b/cve-vulner-manager/cve-py/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..aa988394e8794117035fbf3f9bc08b9c561e732b --- /dev/null +++ b/cve-vulner-manager/cve-py/Dockerfile @@ -0,0 +1,13 @@ +FROM openeuler/openeuler:21.03 +#FROM python:3.8 +LABEL maintainer="zhangjianjun" + +RUN yum install -y gcc +RUN yum -y update && yum install -y python3 && yum install -y python3-pip mysql-devel && yum install -y python3-devel +RUN pip install mysql-connector +RUN mkdir -p /opt/app/cve-python +WORKDIR /opt/app/cve-python +COPY . /opt/app/cve-python/ +RUN pip install --no-cache-dir -r /opt/app/cve-python/requirements.txt + +ENTRYPOINT ["python3", "/opt/app/cve-python/main.py"] \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/README.md b/cve-vulner-manager/cve-py/README.md new file mode 100644 index 0000000000000000000000000000000000000000..38a5af8a24dbf0f7f6a43e6ba41b461cd6266616 --- /dev/null +++ b/cve-vulner-manager/cve-py/README.md @@ -0,0 +1,34 @@ +# cve-artificial + +## Introduction + +This part of the code is to realize the manual data entry module. + +## Architecture + +![](https://images.gitee.com/uploads/images/2020/1019/171406_17619722_7432213.png "Structure.png") + +## Prerequisites + +You'll need to setup a MySQL Database before you are getting started. +This is an example to create Database instance. + +* Setup MySQL instance by the Huawei Cloud Relational Database Service (RDS) +* Login in MySQL with your account and password +* Create database instance by running the following command + ``` + CREATE DATABASE cvevulner; + source ./doc/sql/db_struct.sql; + ``` +The information of database instance will be used in the following Installation. + +## Getting Started + +* [Build the project](doc/md/installx.md) + +## Process processing help + +See the [Process processing help](doc/md/manual.md) file for details. +> For Chinese version, please refer to [流程处理帮助](doc/md/manual.md). + + diff --git a/cve-vulner-manager/cve-py/common/files.py b/cve-vulner-manager/cve-py/common/files.py new file mode 100644 index 0000000000000000000000000000000000000000..23c9e7311fba2e8ba25ba7b499e103311ca593a7 --- /dev/null +++ b/cve-vulner-manager/cve-py/common/files.py @@ -0,0 +1,65 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +common files + +Authors: zhangjianjun +Date: 04/01/2021 11:01 AM +""" + +import os +import shutil +import yaml + + +def mkdirs(file_path, file_flag): + """ + Create a directory, return the creation success or failure, bool value + file_flag ==1 Delete all files + return bool + """ + path = file_path.strip() + path = path.rstrip("/") + isExists = os.path.exists(path) + if not isExists: + try: + os.makedirs(path) + except OSError: + return False + else: + if file_flag == 1: + del_list = os.listdir(path) + for f in del_list: + file_path = os.path.join(path, f) + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + return True + + +def file_isexists(file_path): + """ + Determine whether the file exists + return bool + """ + if os.path.exists(file_path): + return True + else: + return False + + +def parse_yaml(file_name): + """ + Parse yaml + return dict + """ + with open(file_name) as f: + yaml_data = yaml.safe_load(f) + return yaml_data diff --git a/cve-vulner-manager/cve-py/common/times.py b/cve-vulner-manager/cve-py/common/times.py new file mode 100644 index 0000000000000000000000000000000000000000..9dbf3ca36e5df14bbaab7204d377a454ed004d86 --- /dev/null +++ b/cve-vulner-manager/cve-py/common/times.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +common time + +Authors: zhangjianjun +Date: 04/01/2021 11:01 AM +""" + +import time +import datetime + + +def get_current_time(): + """ + get current time + """ + now = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + return now + + +def last_month_date(): + """ + last month date + :return createTime: string + """ + today = datetime.datetime.now() + # 计算偏移量 + offset = datetime.timedelta(days=-30) + # 获取想要的日期的时间 + last_month = (today + offset).strftime('%Y-%m-%d') + print("last_month: ", last_month) + return last_month diff --git a/cve-vulner-manager/cve-py/controller/__init__.py b/cve-vulner-manager/cve-py/controller/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..01fe626a369b92b73546c78ac54c99d5720a2cab --- /dev/null +++ b/cve-vulner-manager/cve-py/controller/__init__.py @@ -0,0 +1,3 @@ +""" + Control task +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/controller/taskcontroller.py b/cve-vulner-manager/cve-py/controller/taskcontroller.py new file mode 100644 index 0000000000000000000000000000000000000000..e999804171c06c52ecf4a20581913c24d425e3b0 --- /dev/null +++ b/cve-vulner-manager/cve-py/controller/taskcontroller.py @@ -0,0 +1,197 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Control task + +Authors: xiaojianghui +update by: zhangjianjun +Date: 10/22/2020 11:01 AM +""" + +from tabletask import (runtask, mappingtask, toexcel, export_excel_task, + import_excel_task, package_committer_task, + spec_error_task, repeattask, gauss_yaml, + issue_statistics, supplement_cve, mindspore_yaml, + openlookeng_yaml) +from gitwebtask import genegroup, yamltask +from emailtask import sendingtask, issue_record_email +from deletetask import deletefiletask +import os +import threading + +mutex_lock = threading.Lock() + +def toexcelcontroller(): + """ + Query the data whose cve_status status is 3 or 4 in the + database to generate excel Modify the status of cve_status to 3 or 4 to 5, + indicating that it has been added to excel to be processed + + Returns: + return None + """ + print("Export data to generate excel, " + "send error data feedback email task start") + toexcel.MysqlToExcel().generate_table() + toexcel.MysqlToExcel().update_status() + sendingtask.sending() + print("To excel and send email task completed") + + +def runtabletask(): + """ + Parse the artificial CVE table, crawl the CVE official + website data, and store it in the database + return None + """ + print("Analyze the manual data table, " + "crawl the CVE official website data, and start the timing task") + spec_error_task.add_error_details() + runtask.handle_data() + repeattask.repeat_task() + repeattask.get_published_date_task() + print("Analyze the manual data table, " + "crawl the CVE official website data task completed") + + +def runmappeingtask(): + """ + Parse the artificial CVE table, crawl the CVE + official website data, and store it in the database + return None + """ + print("The timing task of parsing the " + "package name mapping table starts") + mappingtask.to_mysql() + print("The timing task of parsing the " + "package name mapping table completed") + + +def gwcontroller(): + """ + Grab warehouse yaml data + return None + """ + print("Start of crawling organization, warehouse, " + "owners correspondence relationship timing task") + genegroup.Command().handle() + yamltask.Task().download() + yamltask.Task().read() + print("Grab warehouse yaml data task completed") + + +def rundelfiletask(): + """ + Delete files older than one month + return None + """ + + print("Clean up the folder timed task starts") + deletefiletask.deldir("./oldexcels") + print("Clean up the folder task completed") + + +def run_issue_record_task(): + """ + issues record email task + return None + """ + print('issues record email task start') + for filex in os.listdir('./export_excels'): + os.remove('./export_excels/' + filex) + export_excel_task.generate_excels(1, './export_excels/create_issues') + export_excel_task.generate_excels(2, './export_excels/unctreated_issues') + issue_record_email.sending() + print('issues record email task complete') + + +def run_whiltlist_task(): + """ + the task of parsing package whitelist + return None + """ + print('the task of parsing package whitelist start') + import_excel_task.import_data() + print('the task of parsing package whitelist complete') + + +def run_package_committer_task(): + """ + the task of parsing package_committer excels + Returns:None + + """ + print('the task of parsing package_committer_excels start') + package_committer_task.import_data() + print('the task of parsing package_committer_excels complete') + + +def parse_opengauss_yaml_task(): + """ + Download the yaml file first, then parse the yaml file of opengauss + Returns:None + + """ + print('the task of Process opengauss yaml data start') + gauss_yaml.proc_gauss_yaml() + print('the task of Process opengauss yaml data complete') + + +def issue_statistics_email_task(): + """ + Parse the issue statistics recipient list + return None + """ + print('Parse the issue statistics recipient list start') + issue_statistics.get_issue_excel() + print('Parse the issue statistics recipient list complete') + + +def supplement_cve_task(): + """ + Complete the template information of the + issue with the data on the CVE official website + return None + """ + print("Supplemental cve information task starts") + mutex_lock.acquire() + supplement_cve.supplement_cve() + mutex_lock.release() + print("Supplement cve information task is over") + + +def long_supplement_cve_task(): + """ + Complete the template information of the + issue with the data on the CVE official website + return None + """ + print("long Supplemental cve information task starts") + supplement_cve.long_supplement_cve() + print("long Supplement cve information task is over") + + +def parse_mindspore_yaml_task(): + """ + Download the yaml file first, then parse the yaml file of mindspore + Returns:None + """ + print('the task of Process mindspore yaml data start') + mindspore_yaml.proc_mindspore_yaml() + print('the task of Process mindspore yaml data complete') + + +def parse_openlookeng_yaml_task(): + """ + Download the yaml file first, then parse the yaml file of openLooKeng + Returns:None + """ + print('the task of Process openLooKeng yaml data start') + openlookeng_yaml.proc_openlookeng_yaml() + print('the task of Process openLooKeng yaml data complete') diff --git a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py new file mode 100644 index 0000000000000000000000000000000000000000..429e6e239637971a55758ab1de78002cb2e9f1aa --- /dev/null +++ b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Control task + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import apscheduler.schedulers.background as background +from controller import taskcontroller + + +def timertask(): + """ + Scheduled tasks summary + :return:None + """ + try: + scheduler = background.BackgroundScheduler(job_defaults={'max_instances': 200}) + # Add tasks + # Warehouse, organization, and owner correspondence timing task + scheduler.add_job(taskcontroller.gwcontroller, 'cron', day_of_week='0-6', hour=0, minute=2) + # Manual CVE data entry timing task + scheduler.add_job(taskcontroller.runtabletask, 'interval', minutes=40) + # Error data export and email sending timing task + scheduler.add_job(taskcontroller.toexcelcontroller, 'cron', day_of_week='0-6', hour=7, minute=30) + # Package name correspondence relationship timing task + scheduler.add_job(taskcontroller.runmappeingtask, 'cron', day_of_week='0-6', hour=7, minute=30) + # Delete ,iles timed tasks that are more than one month old + scheduler.add_job(taskcontroller.rundelfiletask, 'cron', day_of_week='0-6', hour=9, minute=30) + # issue record email feedback + scheduler.add_job(taskcontroller.run_issue_record_task, 'cron', day_of_week='0-6', hour=7, minute=30) + # analysis the table package whitelist + scheduler.add_job(taskcontroller.run_whiltlist_task, 'interval', hours=2) + # parsing the table package_committer_excels + scheduler.add_job(taskcontroller.run_package_committer_task, 'interval', hours=1) + # Parse the issue statistics recipient list + # scheduler.add_job(taskcontroller.issue_statistics_email_task, 'cron', day_of_week='0-6', hour=5, minute=30) + # Complete the template information of the issue with the data on the CVE official website + scheduler.add_job(taskcontroller.supplement_cve_task, 'interval', minutes=33) + scheduler.add_job(taskcontroller.long_supplement_cve_task, 'cron', day_of_week='0-6', hour=1, minute=30) + # Parse opengauss yaml file + scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=2, minute=30) + # Parse the yaml file of mindspore + scheduler.add_job(taskcontroller.parse_mindspore_yaml_task, 'cron', day_of_week='0-6', hour=3, minute=30) + # Parse the yaml file of openLooKeng + # scheduler.add_job(taskcontroller.parse_openlookeng_yaml_task, 'cron', day_of_week='0-6', hour=4, minute=30) + scheduler.start() + except SystemExit as err: + print("Err:", err) diff --git a/cve-vulner-manager/cve-py/dbConnecttion/Config.py b/cve-vulner-manager/cve-py/dbConnecttion/Config.py new file mode 100644 index 0000000000000000000000000000000000000000..a7fda1cfd35bc76f5c1578fe4759e4db6437f7f9 --- /dev/null +++ b/cve-vulner-manager/cve-py/dbConnecttion/Config.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Database configuration file + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import os + +DBHOST = os.getenv("DB_URI", "127.0.0.1") +DBPORT = 3306 +#DBPORT = 3310 +DBUSER = "cve" +DBPWD = os.getenv("DB_PWDPY") +DBNAME = "cvevulner" +#DBNAME = "oneline_cvevulner" +DBCHAR = "utf8" diff --git a/cve-vulner-manager/cve-py/dbConnecttion/MysqlConn.py b/cve-vulner-manager/cve-py/dbConnecttion/MysqlConn.py new file mode 100644 index 0000000000000000000000000000000000000000..b1c4032ef7f95ed313e738050b5fc5b79595eee3 --- /dev/null +++ b/cve-vulner-manager/cve-py/dbConnecttion/MysqlConn.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +1. When executing SQL with parameters, please use sql +statement to specify the list of conditions that +need to be entered, and then use tuple/list for conditional batch allocation +2. There is no need to use quotation marks to specify the data type in the format SQL, +and the system will automatically recognize it according to the input parameters +3. There is no need to use conversion function in the input +value, the system will automatically handle it + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" +import pymysql +from pymysql import cursors +from DBUtils import PooledDB +from dbConnecttion import Config + + +class Mysql(object): + """ + Connection pool object + """ + __pool = None + + def __init__(self): + """ + init object + """ + self.__conn = Mysql.__getConn() + self.__cursor = self.__conn.cursor() + + @staticmethod + def __getConn(): + """ + Static method, remove the connection from the connection pool + :return MySQLdb.connection + """ + if Mysql.__pool is None: + __pool = PooledDB.PooledDB(creator=pymysql, mincached=1, + maxcached=20, host=Config.DBHOST, + port=Config.DBPORT, + user=Config.DBUSER, + passwd=Config.DBPWD, db=Config.DBNAME, + use_unicode="UTF-8", charset=Config.DBCHAR, + cursorclass=cursors.DictCursor) + return __pool.connection() + return None + + def getOne(self, sql, param=None): + """ + Execute the query and take out the first + :param sql:Query SQL. If there are query conditions, please specify + only the condition list and pass in the condition value using the parameter [param] + :param param:Optional parameter, condition list value (tuple/list) + :return:result list/boolean Query result set + """ + if param is None: + count = self.__cursor.execute(sql) + else: + count = self.__cursor.execute(sql, param) + if count > 0: + result = self.__cursor.fetchone() + else: + result = False + return result + + def getMany(self, sql, param=None): + """ + Execute the query and take out num results + :param sql:Query SQL. If there are query conditions, please specify + only the condition list and pass in the condition value using the parameter [param] + :param param:Optional parameter, condition list value (tuple/list) + :return:result list/boolean Query result set + """ + if param is None: + count = self.__cursor.execute(sql) + else: + count = self.__cursor.execute(sql, param) + if count > 0: + result = self.__cursor.fetchall() + else: + result = False + return result + + def insertOne(self, sql, value): + """ + Insert a record into the data table + :param sql:SQL format to be inserted + :param value:Record data to be inserted tuple/list + :return:insertId Number of rows affected + """ + self.__cursor.execute(sql, value) + return self.__getInsertId() + + def __getInsertId(self): + """ + Get the id generated by the last insert operation + of the current connection, or 0 if not + """ + self.__cursor.execute("SELECT @@IDENTITY AS id") + result = self.__cursor.fetchall() + return result[0]['id'] + + def __query(self, sql, param=None): + """ + query data table records + :param sql:SQL format and conditions, use (%s,%s) + :param param:The value to be updated tuple/list + :return:count Number of rows affected + """ + if param is None: + count = self.__cursor.execute(sql) + else: + count = self.__cursor.execute(sql, param) + return count + + def update(self, sql, param=None): + """ + Update data table records + :param sql:SQL format and conditions, use (%s,%s) + :param param:The value to be updated tuple/list + :return:count Number of rows affected + """ + return self.__query(sql, param) + + def delete(self, sql, param=None): + """ + Delete data table record + :param sql:SQL format and conditions, use (%s,%s) + :param param:Condition to be deleted Value tuple/list + :return:count Number of rows affected + """ + return self.__query(sql, param) + + def begin(self): + """ + Open transaction + """ + self.__conn.autocommit(0) + + def end(self, option='commit'): + """ + End the transaction + """ + if option == 'commit': + self.__conn.commit() + else: + self.__conn.rollback() + + def dispose(self, is_end=1): + """ + Commit transaction + """ + if is_end == 1: + self.end('commit') + else: + self.end('rollback') + + def close(self): + """ + Close the connection + """ + self.__cursor.close() + self.__conn.close() diff --git a/cve-vulner-manager/cve-py/dbConnecttion/__init__.py b/cve-vulner-manager/cve-py/dbConnecttion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..96c381b2818a9184954ed24ece68565b35eec741 --- /dev/null +++ b/cve-vulner-manager/cve-py/dbConnecttion/__init__.py @@ -0,0 +1,3 @@ +""" + connect mysql +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/deletetask/__init__.py b/cve-vulner-manager/cve-py/deletetask/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9ef0244c48218ab5466df394e45f3855b963e11a --- /dev/null +++ b/cve-vulner-manager/cve-py/deletetask/__init__.py @@ -0,0 +1,3 @@ +""" + Clear historical files regularly +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/deletetask/deletefiletask.py b/cve-vulner-manager/cve-py/deletetask/deletefiletask.py new file mode 100644 index 0000000000000000000000000000000000000000..b1be8108312cfbca11f5682c3e7e8970bd44a0bd --- /dev/null +++ b/cve-vulner-manager/cve-py/deletetask/deletefiletask.py @@ -0,0 +1,51 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Clear historical files regularly + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import datetime +import os +import time + + +def deldir(dirx): + """ + Delete files older than one month + :param dirx:dir struct + """ + files = os.listdir(dirx) + for filex in files: + file_path = dirx + "/" + filex + if os.path.isfile(file_path): + # Last modified time + # Get file timestamp + last1 = os.stat(file_path).st_mtime + filetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(last1)) + # Get expiration time + starttime = datetime.datetime.now() + d1 = starttime + datetime.timedelta(days=-30) + date1 = str(d1) + index = date1.find(".") + datatime01 = date1[:index] + # datatime01 is the time 7 days before the current time, filetime is the time of file modification, + # if the file time is less than (earlier) datatime01 time, delete fileu + if datatime01 > filetime: + os.remove(file_path) + print(file_path + "was removed!") + elif os.path.isdir(file_path): + # If it is a folder, continue to traverse + deldir(file_path) + # Delete if the folder is empty + if not os.listdir(file_path): + os.rmdir(file_path) + print("empty folder" + file_path + "was removed!") diff --git a/cve-vulner-manager/cve-py/downloadtask/__init__.py b/cve-vulner-manager/cve-py/downloadtask/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5af5d40e43e5d59cb571f0028d4485221f137852 --- /dev/null +++ b/cve-vulner-manager/cve-py/downloadtask/__init__.py @@ -0,0 +1,3 @@ +""" + Data update provides download tasks +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/downloadtask/downloadfiletask.py b/cve-vulner-manager/cve-py/downloadtask/downloadfiletask.py new file mode 100644 index 0000000000000000000000000000000000000000..a83ac770ee74797d03da1baee80796c791c7eec6 --- /dev/null +++ b/cve-vulner-manager/cve-py/downloadtask/downloadfiletask.py @@ -0,0 +1,248 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Data update provides download tasks + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import requests +import lxml.etree as etree +import shutil +import os + + +def handle_one(): + """ + download file + """ + if os.listdir("./newexcels"): + shutil.rmtree("./newexcels") + os.mkdir("./newexcels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/newexcels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + local_name = name.lower() + if local_name.endswith(".xls") or local_name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/newexcels/" + name + r = requests.get(url) + print(r.status_code) + with open("./newexcels/" + name, 'wb') as code: + code.write(r.content) + code.close() + if os.path.exists("./newexcels/" + name): + print("Successfully downloaded the cve manual data form:" + name) + i += 2 + + +def handle_two(): + """ + download file + """ + if os.listdir("./mappingexcels"): + shutil.rmtree("./mappingexcels") + os.mkdir("./mappingexcels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/mappingexcels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/mappingexcels/" + name + r = requests.get(url) + print(r.status_code) + with open("./mappingexcels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./mappingexcels/" + name): + print("Download the package name mapping table successfully:" + name) + i += 2 + + +def handle_three(): + """ + download file + """ + if os.listdir("./import_excels"): + shutil.rmtree("./import_excels") + os.mkdir("./import_excels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/import_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/import_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./import_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./import_excels/" + name): + print("Download the Package whitelist table successfully:" + name) + i += 2 + + +def handle_four(): + """ + download file + """ + if os.listdir("./package_committer_excels"): + shutil.rmtree("./package_committer_excels") + os.mkdir("./package_committer_excels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/package_committer_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/package_committer_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./package_committer_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./package_committer_excels/" + name): + print("Download the package_committer table successfully:" + name) + i += 2 + + +def download_spec_error_excels(): + """ + download spec_error_excels + """ + if not os.path.exists('./spec_error_excels'): + os.makedirs('./spec_error_excels') + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/spec_error_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/spec_error_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./spec_error_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./spec_error_excels/" + name): + print("Download the package_committer table successfully:" + name) + i += 2 + + +def download_yaml(file_path, file_url): + """ + Download yaml data from gitee + """ + try: + r = requests.get(file_url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + with open(file_path, "wb") as code: + code.write(r.content) + + +def download_excel(file_dir): + """ + download excel + """ + local_dir = "./" + file_dir + gitee_dir = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/' + file_dir + urlx = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/%s/" % (file_dir) + if os.listdir(local_dir): + shutil.rmtree(local_dir) + os.mkdir(local_dir) + try: + r = requests.get(gitee_dir) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = urlx + name + r = requests.get(url) + print(r.status_code) + with open(local_dir + "/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists(local_dir + "/" + name): + print("Download the excel successfully:" + name) + i += 2 diff --git a/cve-vulner-manager/cve-py/emailtask/__init__.py b/cve-vulner-manager/cve-py/emailtask/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..375f213f622850769e6cd08204460842ea6e505a --- /dev/null +++ b/cve-vulner-manager/cve-py/emailtask/__init__.py @@ -0,0 +1,3 @@ +""" + Email notification of incorrect data +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/emailtask/issue_record_email.py b/cve-vulner-manager/cve-py/emailtask/issue_record_email.py new file mode 100644 index 0000000000000000000000000000000000000000..60c80599dec96af0a48d6ab4fb830d0a36c170d1 --- /dev/null +++ b/cve-vulner-manager/cve-py/emailtask/issue_record_email.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +send email + +Authors: xiaojianghui +Date: 11/06/2020 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +from emailtask import sendemail +import os + + +def sending(): + """ + Query all recipients + """ + mysql = Mysql() + sql = "select email_name from cve_email_list where email_type=1" + result = mysql.getMany(sql) + mysql.close() + print(result) + for i in result: + subject = "This time cve creates an issue record" + content = "The email attachment may contain two attached excel documents, " \ + "which are the issue data that has been successfully " \ + "created and the issue data that has not been created." + # sendemail.send_email("smtp.gmail.com", 587, + # os.getenv("CVE_EMAIL_SENDADDR"), + # os.getenv("CVE_EMAIL_PASSWORD"), + # str(i['email_name']), './export_excels', subject, content) + sendemail.send_email('mailman-exim4-service.mail.svc.cluster.local', 25, + os.getenv("CVE_EMAIL_SENDADDR"), + os.getenv("CVE_EMAIL_PASSWORD"), + str(i['email_name']), './export_excels', subject, content) diff --git a/cve-vulner-manager/cve-py/emailtask/sendemail.py b/cve-vulner-manager/cve-py/emailtask/sendemail.py new file mode 100644 index 0000000000000000000000000000000000000000..d7faf92a9333f4a56a3fdcff4bf0cafb05457bef --- /dev/null +++ b/cve-vulner-manager/cve-py/emailtask/sendemail.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Email notification of incorrect data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import smtplib +import os +import email.mime.multipart +import email.mime.text +import email.mime.application as application +import email.utils + + +def send_email(smtp_host, smtp_port, send_addr, password, recipient_addrs, path, subject='', content=''): + """ + send email + :param smtp_host:smpt.gmail.com + :param smtp_port:587 + :param send_addr: + :param password: + :param recipient_addrs: + :param path: + :param subject: title + :param content: content + :return:None + """ + msg = email.mime.multipart.MIMEMultipart() + msg['from'] = email.utils.formataddr(["cve-manager", send_addr]) + msg['to'] = recipient_addrs + msg['subject'] = subject + content = content + txt = email.mime.text.MIMEText(content, 'plain', 'utf-8') + msg.attach(txt) + # Add attachment address + files = os.listdir(path) + if not files: + print("No data sheet") + return + for fileName in files: + part = application.MIMEApplication(open(path + '/' + fileName, 'rb').read()) + part.add_header('Content-Disposition', 'attachment', filename=fileName) + msg.attach(part) + try: + smtp_ssl_client = smtplib.SMTP(smtp_host, smtp_port) + # smtp_ssl_client.ehlo() + # smtp_ssl_client.starttls() + # login_res = smtp_ssl_client.login(send_addr, password) + # print("Login result:login_res=", login_res) + # if login_res and login_res[0] == 235: + # print("login successful，code=[login_res[0]]") + # smtp_ssl_client.sendmail(send_addr, recipient_addrs, str(msg)) + # print("mail has been send successfully. message: ", str(msg)) + # smtp_ssl_client.quit() + # else: + # print("login failed,code= ", login_res[0]) + smtp_ssl_client.sendmail(send_addr, recipient_addrs, str(msg)) + print("mail has been send successfully. message: ", str(msg)) + smtp_ssl_client.quit() + except SystemExit as e: + print("Failed to send,Exception:e= ", e) diff --git a/cve-vulner-manager/cve-py/emailtask/sendingtask.py b/cve-vulner-manager/cve-py/emailtask/sendingtask.py new file mode 100644 index 0000000000000000000000000000000000000000..2a421538a16cbfe25ecdaaa390ee5f4c2d566a48 --- /dev/null +++ b/cve-vulner-manager/cve-py/emailtask/sendingtask.py @@ -0,0 +1,36 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Email notification of incorrect data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import os +from dbConnecttion import MysqlConn +from emailtask import sendemail + + +def sending(): + """ + Query all recipients + """ + mysql = MysqlConn.Mysql() + sql = "select email_name from cve_email_list where email_type=1" + result = mysql.getMany(sql) + mysql.close() + print(result) + for i in result: + subject = "Form error data feedback(人工CVE漏洞数据错误反馈)" + content = "The submitted CVE data is wrong, please check the format and content, please refer to the attachment" + sendemail.send_email('mailman-exim4-service.mail.svc.cluster.local', 25, + os.getenv("CVE_EMAIL_SENDADDR"), + os.getenv("CVE_EMAIL_PASSWORD"), + str(i['email_name']), './problemexcels', subject, content) diff --git a/cve-vulner-manager/cve-py/export_excels/test.xls b/cve-vulner-manager/cve-py/export_excels/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cve-vulner-manager/cve-py/gauss_yaml/gauss_yaml.yaml b/cve-vulner-manager/cve-py/gauss_yaml/gauss_yaml.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16fcbc06e4aeec47abb2f7472bad1ef29ecaf7ac --- /dev/null +++ b/cve-vulner-manager/cve-py/gauss_yaml/gauss_yaml.yaml @@ -0,0 +1,252 @@ +jemalloc: + cpeName: jemalloc + version: 5.2.1 + url: 'https://github.com/jemalloc' +libedit: + cpeName: libedit + version: 3.1 + url: 'http://thrysoee.dk/editline/' +OpenSSL: + cpeName: OpenSSL + version: 1.1.1d + url: 'https://www.openssl.org/news/vulnerabilities.html' +lz4: + cpeName: lz4 + version: v1.9.2 + url: 'https://github.com/lz4' +Postgresql JDBC Driver: + cpeName: Postgresql JDBC Driver + version: 42.2.5 + url: 'https://jdbc.postgresql.org/index.html' +Postgres-XC: + cpeName: Postgres-XC + version: 1.1 + url: '' +PSQL ODBC: + cpeName: psqlodbc + version: 10.03.0000 + url: 'https://www.postgresql.org/' +snappy: + cpeName: snappy + version: 1.1.7 + url: '' +protobuf: + cpeName: protobuf + version: 3.7.1 + url: 'https://github.com/protocolbuffers/protobuf' +masstree: + cpeName: masstree + version: v0.9.0 + url: '' +mysql_fdw: + cpeName: mysql_fdw + version: v2.5.3 + url: 'https://github.com/EnterpriseDB/mysql_fdw' +oracle_fdw: + cpeName: oracle_fdw + version: v2.2.0 + url: 'https://github.com/laurenz/oracle_fdw' +llvm: + cpeName: llvm + version: 7.0.0 + url: 'https://github.com/llvm' +ncurses: + cpeName: ncurses + version: 6.2 + url: '' +paramiko: + cpeName: paramiko + version: 2.6.0 + url: 'https://github.com/paramiko/paramiko' +huaweicloud-sdk-c-obs: + cpeName: huaweicloud-sdk-c-obs + version: 3.1.3 + url: 'https://github.com/huaweicloud/huaweicloud-sdk-c-obs' +Kerberos 5 (KRB5): + cpeName: Kerberos 5 (KRB5) + version: 1.17.1 + url: 'https://kerberos.org/' +Curl and Libcurl: + cpeName: Curl and Libcurl + version: 7.68.0 + url: 'https://curl.se/docs/security.html' +orc: + cpeName: orc + version: 1.5.7 + url: 'https://orc.apache.org/security/' +libiconv: + cpeName: libiconv + version: 1.16 + url: 'https://www.gnu.org/software/libiconv/' +pcre: + cpeName: pcre + version: 8.42 + url: 'https://www.pcre.org/' +psutil: + cpeName: psutil + version: 5.6.7 + url: 'https://pypi.org/project/psutil/' +Netifaces: + cpeName: Netifaces + version: 0.10.9 + url: 'https://pypi.org/project/netifaces/' +nghttp2: + cpeName: nghttp2 + version: 1.39.2 + url: 'https://github.com/nghttp2/nghttp2' +'python::cryptography': + cpeName: python::cryptography + version: 2.7 + url: 'https://pypi.org/project/cryptography/' +'python::cffi': + cpeName: python::cffi + version: 1.12.3 + url: 'https://pypi.org/project/cffi/' +pycparser: + cpeName: pycparser + version: 2.19 + url: 'https://pypi.org/project/pycparser/' +'python::asn1crypto': + cpeName: python::asn1crypto + version: 1.2.0 + url: 'https://pypi.org/project/asn1crypto/' +idna: + cpeName: idna + version: 2.8 + url: 'https://pypi.org/project/idna/' +six: + cpeName: six + version: 1.12.0 + url: 'https://pypi.org/project/six/' +ipaddress: + cpeName: ipaddress + version: 1.0.22 + url: 'https://pypi.org/project/ipaddress/' +pyasn1: + cpeName: pyasn1 + version: 0.4.7 + url: 'https://pypi.org/project/pyasn1/' +PyNaCl: + cpeName: PyNaCl + version: 1.3.0 + url: 'https://pypi.org/project/PyNaCl/' +bcrypt: + cpeName: bcrypt + version: 3.1.7 + url: 'https://pypi.org/project/bcrypt/' +zlib: + cpeName: zlib + version: 1.2.11 + url: 'https://github.com/madler/zlib' +postgresql-hll: + cpeName: postgresql-hll + version: 2.10.2 + url: 'https://github.com/citusdata/postgresql-hll' +numactl: + cpeName: numactl + version: 2.0.13 + url: 'https://github.com/numactl/numactl' +pyOpenSSL: + cpeName: pyOpenSSL + version: 19.0.0 + url: 'https://pypi.org/project/pyOpenSSL/' +Apache Arrow: + cpeName: Apache Arrow + version: v0.11.1 + url: 'https://github.com/apache/arrow' +Apache Thrift: + cpeName: Apache Thrift + version: 0.13.0 + url: 'https://github.com/apache/thrift' +boost: + cpeName: boost + version: 1.71.0 + url: 'https://www.boost.org/' +brotli: + cpeName: brotli + version: v0.6.0 + url: 'https://github.com/google/brotli' +c-ares: + cpeName: c-ares + version: 1.15.0 + url: 'https://c-ares.haxx.se/vulns.html' +double-conversion: + cpeName: double-conversion + version: 3.1.1 + url: 'https://github.com/google/double-conversion' +flatbuffers: + cpeName: flatbuffers + version: 1.11.0 + url: 'https://github.com/google/flatbuffers' +google/glog: + cpeName: google/glog + version: 0.4.0 + url: 'https://github.com/google/glog' +grpc: + cpeName: grpc + version: 1.22.0 + url: 'https://github.com/grpc/grpc' +libcgroup: + cpeName: libcgroup + version: 0.41-21.el7 + url: 'https://github.com/matsumotory/libcgroup' +libxml2: + cpeName: libxml2 + version: 2.9.9 + url: 'https://github.com/GNOME/libxml2' +rapidjson: + cpeName: rapidjson + version: 4b3d7c2f42142f10b888e580c515f60ca98e2ee9 + url: 'http://rapidjson.org/' +Zstandard: + cpeName: Zstandard + version: 1.4.4 + url: 'https://github.com/facebook/zstd' +cJSON: + cpeName: cJSON + version: 1.7.11 + url: 'https://github.com/DaveGamble/cJSON' +libevent: + cpeName: libevent + version: 2.1.11 + url: 'https://github.com/libevent/libevent' +p7zip: + cpeName: p7zip + version: 16.02 + url: 'https://www.7-zip.org/' +gcc: + cpeName: gcc + version: 8.2.0 + url: 'https://gcc.gnu.org/' +gmp: + cpeName: gmp + version: 6.1.2 + url: 'https://gmplib.org/' +mpc: + cpeName: mpc + version: 1.1.0 + url: 'http://ftp.gnu.org/' +mpfr: + cpeName: mpfr + version: 4.0.2 + url: 'https://www.mpfr.org/' +bounds_checking_function: + cpeName: bounds_checking_function + version: v1.1.10 + url: 'https://gitee.com/openeuler' +isl: + cpeName: isl + version: 0.18 + url: 'https://gcc.gnu.org/' +AdoptOpenJDK: + cpeName: AdoptOpenJDK + version: jdk8u222-b10 + url: 'https://adoptopenjdk.net/' +Apache Maven: + cpeName: Apache Maven + version: 3.6.3 + url: 'https://maven.apache.org/security.html' +unixODBC: + cpeName: unixODBC + version: 2.3.6 + url: 'http://www.unixodbc.org/' \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/gitwebtask/__init__.py b/cve-vulner-manager/cve-py/gitwebtask/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9f2d602975663762a08696eda0458d1ff3bc6cba --- /dev/null +++ b/cve-vulner-manager/cve-py/gitwebtask/__init__.py @@ -0,0 +1,3 @@ +""" + The maintainer for crawling yaml data +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/gitwebtask/genegroup.py b/cve-vulner-manager/cve-py/gitwebtask/genegroup.py new file mode 100644 index 0000000000000000000000000000000000000000..8b91186253240581090ebc92764c3768be011e66 --- /dev/null +++ b/cve-vulner-manager/cve-py/gitwebtask/genegroup.py @@ -0,0 +1,99 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +The maintainer for crawling yaml data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import requests +import lxml.etree as etree +import dbConnecttion.MysqlConn as MysqlConn +import time + + +class Command(object): + """ + download sig + """ + + def __init__(self): + """ + init object + """ + pass + + @staticmethod + def handle(): + """ + download detail + """ + url = 'https://gitee.com/openeuler/community/tree/master/sig' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + sigs_list = [] + i = 3 + mysql = MysqlConn.Mysql() + while True: + sig_name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + if sig_name == 'sigs.yaml': + break + sql = "select * from cve_git_repo_groups where group_name = %s" + val = (sig_name,) + flag = mysql.getOne(sql, val) + if not flag: + print("insert data:{}".format(sig_name)) + sql = "insert into cve_git_repo_groups (group_name) values (%s)" + val = (sig_name,) + mysql.insertOne(sql, val) + mysql.dispose() + sig_page = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@href".format(i))[0] + etherpad = 'https://etherpad.openeuler.org/p/{}-meetings'.format(sig_name) + # Get the name, homepage and etherpad of all sigs + sigs_list.append([sig_name, 'https://gitee.com' + sig_page, etherpad]) + i += 2 + sigs_list = sorted(sigs_list) + print(sigs_list) + for sig in sigs_list: + # Gtt group_id + sql = "select group_id from cve_git_repo_groups where group_name = %s" + val = (sig[0],) + group_id = mysql.getOne(sql, val)["group_id"] + # Delete the data in cve_gite_repo_member + # group_id + sql1 = "delete from cve_gite_repo_member where group_id = %s" + val1 = (group_id,) + mysql.delete(sql1, val1) + mysql.dispose() + # Get owners + url = 'https://gitee.com/openeuler/community/blob/master/sig/{}/OWNERS'.format(sig[0]) + r = requests.get(url) + html = etree.HTML(r.text) + res = html.xpath('//div[@class="line"]/text()') + owners = [] + for i in res[1:]: + maintainer = i.strip().split('-')[-1].strip() + create_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + sql = "insert into cve_gite_repo_member (group_id,member_name," \ + "member_type,create_time) values (%s, %s, %s, %s)" + val = (group_id, maintainer, "Maintainer", create_time) + mysql.insertOne(sql, val) + mysql.dispose() + owners.append(maintainer) + print(owners) + mysql.close() diff --git a/cve-vulner-manager/cve-py/gitwebtask/yamltask.py b/cve-vulner-manager/cve-py/gitwebtask/yamltask.py new file mode 100644 index 0000000000000000000000000000000000000000..ad939a475d7dc8aa5ee1973c826b8ea53a6d55a8 --- /dev/null +++ b/cve-vulner-manager/cve-py/gitwebtask/yamltask.py @@ -0,0 +1,80 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +The maintainer for crawling yaml data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import yaml +import requests +from dbConnecttion import MysqlConn +import time +import os + + +class Task(object): + """ + download task for sig + """ + + def __init__(self): + """ + init object + """ + pass + + @staticmethod + def download(): + """ + download detail + """ + url = "https://gitee.com/openeuler/community/raw/master/sig/sigs.yaml" + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + with open("sigs.yaml", "wb") as code: + code.write(r.content) + + @staticmethod + def read(): + """ + read file + """ + if os.path.exists("sigs.yaml"): + with open("sigs.yaml", "r") as f: + data = yaml.safe_load(f) + if data and data['sigs']: + mysql = MysqlConn.Mysql() + for sig in data['sigs']: + sql = "select group_id from cve_git_repo_groups where group_name=%s" + val = (sig['name'],) + group_id = mysql.getOne(sql, val)['group_id'] + # Delete data according to group_id to avoid duplication + sql1 = "delete from cve_gite_repo where group_id = %s" + val1 = (group_id,) + mysql.delete(sql1, val1) + mysql.dispose() + for p in sig["repositories"]: + org_path = p[0:p.rfind("/")] + path = p[p.rfind("/"):][1:] + create_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + sql = "insert into cve_gite_repo (group_id, org_path, path, status, " \ + "create_time) values (%s, %s, %s, %s, %s)" + val = (group_id, org_path, path, 0, create_time) + mysql.insertOne(sql, val) + mysql.dispose() + mysql.close() + os.remove("sigs.yaml") diff --git a/cve-vulner-manager/cve-py/import_excels/26_whitelist_2020-11-26.xlsx b/cve-vulner-manager/cve-py/import_excels/26_whitelist_2020-11-26.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..98eca1855bd0e96b11792afdb999d92da80845ca Binary files /dev/null and b/cve-vulner-manager/cve-py/import_excels/26_whitelist_2020-11-26.xlsx differ diff --git a/cve-vulner-manager/cve-py/import_excels/LTS-SP1-2020-12-29.xlsx b/cve-vulner-manager/cve-py/import_excels/LTS-SP1-2020-12-29.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..acb93225dd89cb7eb53e075a5ea118d29bd2f1c6 Binary files /dev/null and b/cve-vulner-manager/cve-py/import_excels/LTS-SP1-2020-12-29.xlsx differ diff --git a/cve-vulner-manager/cve-py/issue_statistics/~$to_email.xlsx b/cve-vulner-manager/cve-py/issue_statistics/~$to_email.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1e0c8b3a64122aeea8ded50bd91585a6113ab2a6 Binary files /dev/null and b/cve-vulner-manager/cve-py/issue_statistics/~$to_email.xlsx differ diff --git a/cve-vulner-manager/cve-py/main.py b/cve-vulner-manager/cve-py/main.py new file mode 100644 index 0000000000000000000000000000000000000000..efe236af73ded4337102b055923b2e2cfb904a23 --- /dev/null +++ b/cve-vulner-manager/cve-py/main.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" + +1. Process manual excel data; +2. Crawling the maintainer of the warehouse; +3. Initialize the database file; + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" +import flask +from flask import request +from controller import timertaskcontroller +from tabletask import supplement_cve +from gevent import pywsgi +import threading + +app = flask.Flask(__name__) + + +@app.route('/pulls/cve/info', methods=['post']) +def pull_cve_info(): + """ + 1. Receive messages + 2. Grab cve data + """ + cve_num = request.json.get("CveNum") + if cve_num and len(cve_num) > 0: + cve_num = str(cve_num).strip() + ret_value = supplement_cve.pull_cve(cve_num) + if ret_value == 1: + return { + "code": 200, + "message": "success" + } + else: + return { + "code": 404, + "message": "Data does not exist" + } + else: + return { + "code": 405, + "message": "Parameter error" + } + + +def api_proc(): + """ + 1. api thread processing function + """ + server = pywsgi.WSGIServer(('0.0.0.0', 8080), app) + server.serve_forever() + + +if __name__ == '__main__': + print("Start a scheduled task...") + thread_timed_task = threading.Thread(target=timertaskcontroller.timertask) + thread_timed_task.start() + thread_timed_task.join() + thread_api_proc = threading.Thread(target=api_proc) + thread_api_proc.start() + thread_api_proc.join() diff --git "a/cve-vulner-manager/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xlsx" "b/cve-vulner-manager/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xlsx" new file mode 100644 index 0000000000000000000000000000000000000000..ab055ef95795f0ecad0368dea04577fde0a51246 Binary files /dev/null and "b/cve-vulner-manager/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xlsx" differ diff --git "a/cve-vulner-manager/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\2732020-11-06.xlsx" "b/cve-vulner-manager/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\2732020-11-06.xlsx" new file mode 100644 index 0000000000000000000000000000000000000000..ef151ee6a1f7347d614d293ce21c1884b8be48a7 Binary files /dev/null and "b/cve-vulner-manager/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\2732020-11-06.xlsx" differ diff --git a/cve-vulner-manager/cve-py/newexcels/CVE-2021-01-27_1.xlsx b/cve-vulner-manager/cve-py/newexcels/CVE-2021-01-27_1.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..38ea9e1c6539739578fbd7a8c7d445c9275ca3ee Binary files /dev/null and b/cve-vulner-manager/cve-py/newexcels/CVE-2021-01-27_1.xlsx differ diff --git a/cve-vulner-manager/cve-py/oldexcels/test.xls b/cve-vulner-manager/cve-py/oldexcels/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git "a/cve-vulner-manager/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" "b/cve-vulner-manager/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" new file mode 100644 index 0000000000000000000000000000000000000000..efdab9447513e025e5efa1d38ef821f9ca36b3b3 Binary files /dev/null and "b/cve-vulner-manager/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" differ diff --git a/cve-vulner-manager/cve-py/problemexcels/test.xls b/cve-vulner-manager/cve-py/problemexcels/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cve-vulner-manager/cve-py/requirements.txt b/cve-vulner-manager/cve-py/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a43fe40ee07f26dbead13aa75be4e92cac065877 --- /dev/null +++ b/cve-vulner-manager/cve-py/requirements.txt @@ -0,0 +1,29 @@ +APScheduler==3.6.3 +DBUtils==1.3 +Flask==1.1.2 +Flask-SQLAlchemy==2.4.4 +Jinja2==2.11.2 +MarkupSafe==1.1.1 +PyMySQL==0.10.1 +PyYAML==5.3.1 +SQLAlchemy==1.3.20 +Werkzeug==1.0.1 +certifi==2020.6.20 +chardet==3.0.4 +click==7.1.2 +idna==2.10 +itsdangerous==1.1.0 +lxml==4.6.1 +mysql==0.0.3 +mysql-connector-python==8.0.22 +mysqlclient==2.0.1 +protobuf==3.13.0 +pytz==2020.1 +requests==2.24.0 +setuptools==47.1.0 +six==1.15.0 +tzlocal==2.1 +urllib3==1.25.11 +xlrd==1.2.0 +xlwt==1.3.0 +gevent==21.8.0 \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/spec_error_excels/CVE_2021-02-05.xlsx b/cve-vulner-manager/cve-py/spec_error_excels/CVE_2021-02-05.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..ddf8e6cdd5ff58fc62bcc30811fa0f12a9601253 Binary files /dev/null and b/cve-vulner-manager/cve-py/spec_error_excels/CVE_2021-02-05.xlsx differ diff --git a/cve-vulner-manager/cve-py/tabletask/__init__.py b/cve-vulner-manager/cve-py/tabletask/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8f3049bfcc384656f39a9d4dbfdd724a021d6d4f --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/__init__.py @@ -0,0 +1,3 @@ +""" + Interaction between file and mysql data +""" \ No newline at end of file diff --git a/cve-vulner-manager/cve-py/tabletask/common.py b/cve-vulner-manager/cve-py/tabletask/common.py new file mode 100644 index 0000000000000000000000000000000000000000..5cd09f8ee68499149637d74c95ddcf0fdb06d979 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/common.py @@ -0,0 +1,113 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle common tasks + +Authors: zhangjianjun +Date: 05/08/2021 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql + + +def select_openeuler_yaml_data(mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler " \ + "WHERE package_id >= %s" + val = (10000000,) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def delete_yaml_openeuler_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_open_euler where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def delete_yaml_openeuler_detail_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_package_info where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def select_mindspore_yaml_origin_data(package_name, version, mysql): + """ + Query origin data + """ + sql = "SELECT id FROM cve_mind_spore_yaml WHERE package_name = %s " \ + "and version = %s" + val = (package_name, version) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def select_gauss_yaml_origin_data(package_name, version, mysql): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_guss_yaml WHERE package_name = %s " \ + "and version = %s" + val = (package_name, version) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def select_openlookeng_yaml_origin_data(package_name, version, mysql): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = %s " \ + "and version = %s" + val = (package_name, version) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def delete_yaml_data(): + """ + delete yaml data + """ + mysql = Mysql() + # Delete historical data + packages_data = select_openeuler_yaml_data(mysql) + if packages_data is not None: + for pk in packages_data: + print("pk: ", pk) + if pk["package_id"] >= 10000000: + gauss_data = select_gauss_yaml_origin_data(pk["package_name"], pk["version"], mysql) + spore_data = select_mindspore_yaml_origin_data(pk["package_name"], pk["version"], mysql) + lookeng_data = select_openlookeng_yaml_origin_data(pk["package_name"], pk["version"], mysql) + if not gauss_data and not spore_data and not lookeng_data: + print("ID of the currently deleted data: ", pk["git_id"]) + delete_yaml_openeuler_detail_data(mysql, pk["git_id"]) + delete_yaml_openeuler_data(mysql, pk["git_id"]) + mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/crawltask.py b/cve-vulner-manager/cve-py/tabletask/crawltask.py new file mode 100644 index 0000000000000000000000000000000000000000..f546cb9cf73227836689f9f08f91c880b9bb2d77 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/crawltask.py @@ -0,0 +1,202 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import requests +from lxml import etree +from datetime import datetime + + +def crawling(url): + """ + Grab cve specific information + :param url: string + :return xpth_list: list + """ + xpth_list = [] + if url is None or url == "" or url.find("http") == -1: + print("crawling, url:", url) + return + try: + content = requests.get(url).content + except requests.exceptions.ConnectionError: + print('ConnectionError') + return [] + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return [] + if content and len(content) > 1: + html = etree.HTML(content) + try: + # if html.xpath( + # '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[3]/div[2]/div[1]' + # '/div[2]/span/span/a/text()') == [ + # "N/A"] or \ + # html.xpath( + # '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[2]/div[2]/div[1]' + # '/div[2]/span/span/a/text()') == [ + # "N/A"] or \ + # html.xpath( + # '/html/body/div[2]/div[2]/div[2]/table/tr/td/div/div[1]/div[4]/div[2]/div[1]/div[2]' + # '/span/span/a/text()') == ['N/A']: + # if html.xpath( + # "/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[2]/div[3]/div[1]/div[2]" + # "/span/span/a/text()") == [ + # "N/A"] or \ + # html.xpath( + # "/html/body/div[2]/div[2]/div/table/tbody/tr/td/div/div[1]/div[2]/div[2]/div[1]" + # "/div[2]/span/span/a/text()") == ["N/A"]: + # nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \ + # access_vector = attack_complexity = access_complexity = \ + # privilege_required = user_interaction = scope = confidentiality = \ + # integrity = availability = authentication = None + # print("No data on this vulnerability link, ", url) + # score_type = "" + # cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0]) + # if cve_desc: + # score_type = "v3.0" + # else: + # score_type = "v2.0" + # element = html.xpath('//*[@id="nistV2MetricHidden"]/@value') + # cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0]) + # repair_time = str( + # html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[2]/div/span[1]/text()')[0]) + # if repair_time is not None: + # repair_time = datetime.strptime(repair_time, '%m/%d/%Y') + # html1 = etree.HTML(element[0]) + # cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score-severity"]/text()') + # [0].strip()).capitalize() + # nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score"]/text()')[0].strip()) + # vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv2-vector"]/text()')[0]). \ + # replace("(", "").replace(")", "").strip() + # access_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv2-av"]/text()')[0].strip()) + # access_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-ac"]/text()')[0].strip()) + # authentication = str(html1.xpath('//*[@data-testid="vuln-cvssv2-au"]/text()')[0].strip()) + # confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip()) + # integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-i"]/text()')[0].strip()) + # availability = str(html1.xpath('//*[@data-testid="vuln-cvssv2-a"]/text()')[0].strip()) + # attack_vector = attack_complexity = privilege_required = user_interaction = scope = None + # elif html.xpath( + # '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[3]/div[2]/div[1]/div[2]' + # '/span/span/a/text()') == [] and \ + # html.xpath( + # '/html/body/div[2]/div[2]/div/table/tr/td/div/div[1]/div[2]/div[2]/div[1]' + # '/div[2]/span/span/a/text()') == []: + # nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \ + # access_vector = attack_complexity = access_complexity = \ + # privilege_required = user_interaction = scope = confidentiality = integrity = \ + # availability = authentication = None + # score_type = "v3.0" + # print("This vulnerability link not found, ", url) + # else: + # score_type = "v3.0" + # cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0]) + # repair_time = html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[2]/div/span[1]/text()')[0] + # if repair_time is not None: + # repair_time = datetime.strptime(repair_time, '%m/%d/%Y') + # if html.xpath('//*[@id="nistV3MetricHidden"]/@value'): + # element = html.xpath('//*[@id="nistV3MetricHidden"]/@value') + # else: + # element = html.xpath('//*[@id="cnaV3MetricHidden"]/@value') + # html1 = etree.HTML(element[0]) + # cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score-severity"]/text()')[0] + # .strip()).capitalize() + # nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score"]/text()')[0].strip()) + # vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv3-vector"]/text()')[0]).replace("(", ''). \ + # replace(')', '').strip() + # attack_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv3-av"]/text()')[0].strip()) + # attack_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ac"]/text()')[0].strip()) + # privilege_required = str(html1.xpath('//*[@data-testid="vuln-cvssv3-pr"]/text()')[0].strip()) + # user_interaction = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ui"]/text()')[0].strip()) + # scope = str(html1.xpath('//*[@data-testid="vuln-cvssv3-s"]/text()')[0].strip()) + # confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip()) + # integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-i"]/text()')[0].strip()) + # availability = str(html1.xpath('//*[@data-testid="vuln-cvssv3-a"]/text()')[0].strip()) + # access_vector = access_complexity = authentication = None + nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \ + access_vector = attack_complexity = access_complexity = \ + privilege_required = user_interaction = scope = confidentiality = integrity = \ + availability = authentication = None + cve_descx = html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()') + if cve_descx is not None and len(cve_descx) > 0: + cve_desc = str(cve_descx[0]) + # repair_timex = html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[2]/div/span[1]/text()') + repair_timex = html.xpath('//*[@data-testid="vuln-published-on"]/text()') + if repair_timex is not None and len(repair_timex) > 0: + repair_time = str(repair_timex[0]) + if repair_time is not None and repair_time != "": + repair_time = str(datetime.strptime(repair_time, '%m/%d/%Y')) + score_type = "v3.0" + if html.xpath('//*[@id="nistV3MetricHidden"]/@value'): + element = html.xpath('//*[@id="nistV3MetricHidden"]/@value') + else: + element = html.xpath('//*[@id="cnaV3MetricHidden"]/@value') + if element and len(element) > 0: + html1 = etree.HTML(element[0]) + if html1 is not None: + cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score-severity"]/text()')[0] + .strip()).capitalize() + nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score"]/text()')[0].strip()) + vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv3-vector"]/text()')[0]).replace("(", + ''). \ + replace(')', '').strip() + attack_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv3-av"]/text()')[0].strip()) + attack_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ac"]/text()')[0].strip()) + privilege_required = str(html1.xpath('//*[@data-testid="vuln-cvssv3-pr"]/text()')[0].strip()) + user_interaction = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ui"]/text()')[0].strip()) + scope = str(html1.xpath('//*[@data-testid="vuln-cvssv3-s"]/text()')[0].strip()) + confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip()) + integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-i"]/text()')[0].strip()) + availability = str(html1.xpath('//*[@data-testid="vuln-cvssv3-a"]/text()')[0].strip()) + access_vector = access_complexity = authentication = None + else: + element = html.xpath('//*[@id="nistV2MetricHidden"]/@value') + if element and len(element) > 0: + html1 = etree.HTML(element[0]) + if html1 is not None: + score_type = "v2.0" + cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score-severity"]/text()') + [0].strip()).capitalize() + nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score"]/text()')[0].strip()) + vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv2-vector"]/text()')[0]). \ + replace("(", "").replace(")", "").strip() + access_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv2-av"]/text()')[0].strip()) + access_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-ac"]/text()')[0].strip()) + authentication = str(html1.xpath('//*[@data-testid="vuln-cvssv2-au"]/text()')[0].strip()) + confidentiality = str(html1.xpath('//*[@data-testid="vuln-cvssv3-c"]/text()')[0].strip()) + integrity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-i"]/text()')[0].strip()) + availability = str(html1.xpath('//*[@data-testid="vuln-cvssv2-a"]/text()')[0].strip()) + attack_vector = attack_complexity = privilege_required = user_interaction = scope = None + if cve_desc == 'N/A': + cve_desc = None + if repair_time == 'N/A': + repair_time = None + if nvd_score is None or nvd_score == "" or nvd_score == 'N/A': + nvd_score = None + print("nvd_score:", nvd_score, "\n", "cve_level:", cve_level, "\n", + "repair_time:", repair_time, "\n", "score_type:", score_type, "\n", + "vector_value, attack_vector, access_vector,attack_complexity, \n" + "access_complexity, privilege_required, user_interaction, scope,\n" + "confidentiality, integrity, availability, authentication:\n", + vector_value, attack_vector, access_vector, + attack_complexity, access_complexity, privilege_required, user_interaction, scope, + confidentiality, integrity, availability, authentication, "\n", "cve_desc:", cve_desc) + xpth_list = [nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, access_vector, + attack_complexity, access_complexity, privilege_required, user_interaction, scope, + confidentiality, integrity, availability, authentication, score_type] + except IndexError as e: + print("Subscript out of bounds", e) + except UnboundLocalError as e: + print("Tag not found", e) + return xpth_list diff --git a/cve-vulner-manager/cve-py/tabletask/exceltask.py b/cve-vulner-manager/cve-py/tabletask/exceltask.py new file mode 100644 index 0000000000000000000000000000000000000000..b263ac879bf5284fedf19a644e70e75e8b567761 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/exceltask.py @@ -0,0 +1,121 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import xlrd +import os +import shutil + + +def crawl_urls(file_name): + """ + Read excel table content and CVE score + :param file_name: excel file + :return urls: list + """ + urls = [] + try: + data = xlrd.open_workbook("./newexcels/" + file_name) + # Find the vulnerability report information table by name + table_one = data.sheet_by_name("Sheet1") + # Get the maximum number of rows of all data + row_number = table_one.nrows + for i in range(1, row_number): + url = "https://nvd.nist.gov/vuln/detail/" + str(table_one.cell(i, 2).value) + urls.append(url) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return urls + + +def crawl_cve_num(file_name): + """ + CVE number + :param file_name:excel file + :return cve_num:list + """ + cve_num = [] + try: + data = xlrd.open_workbook("./newexcels/" + file_name) + # Find the vulnerability report information table by name + table_one = data.sheet_by_name("Sheet1") + # Get the maximum number of rows of all data + row_number = table_one.nrows + for i in range(1, row_number): + cve_num_one = str(table_one.cell(i, 2).value) + cve_num.append(cve_num_one) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return cve_num + + +def crawl_cve_version(file_name): + """ + CVE version + :param file_name:excel file + :return cve_version:list + """ + cve_version = [] + try: + data = xlrd.open_workbook("./newexcels/" + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + cve_version_one = table_one.cell(i, 1).value + cve_version.append(cve_version_one) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return cve_version + + +def crawl_packname(file_name): + """ + Component + :param file_name:excel file + :return pack_name:string + """ + pack_name = [] + try: + data = xlrd.open_workbook("./newexcels/" + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + pack_name_one = table_one.cell(i, 0).value + pack_name.append(pack_name_one) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return pack_name + + +def move_file(file_name): + """ + excel After the analysis is complete, call the function to move to the oldexcels folder for backup + :param file_name:excel file + """ + try: + f_src = os.path.join("./newexcels/" + file_name) + if not os.path.isdir("./oldexcels/"): + os.mkdir("./oldexcels/") + f_dst = os.path.join("./oldexcels/" + file_name) + shutil.move(f_src, f_dst) + except SystemExit as e: + print("move error", e) diff --git a/cve-vulner-manager/cve-py/tabletask/export_excel_task.py b/cve-vulner-manager/cve-py/tabletask/export_excel_task.py new file mode 100644 index 0000000000000000000000000000000000000000..6503e46288ef469b9683391b9460ef484b83ab85 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/export_excel_task.py @@ -0,0 +1,104 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +export data from mysql + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import time +import xlwt +from dbConnecttion.MysqlConn import Mysql +from common import times + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d", time.localtime()) + return create_time + + +def get_results(status): + """ + Execute mysql + :param status: int + :return: list + """ + mysql = Mysql() + last_month = times.last_month_date() + sql = "select * from cve_issue_create_record where status = %s and create_time > %s" + val = (status, last_month) + results = mysql.getMany(sql, val) + print(results) + mysql.dispose() + mysql.close() + return results + + +def update_status(status): + """ + Execute mysql + :param status: int + :return: list + """ + mysql = Mysql() + sql = 'update cve_issue_create_record set status = %s,update_time = %s where status = %s' + val = (4, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), status) + mysql.update(sql, val) + mysql.dispose() + mysql.close() + + +def generate_excels(status, path): + """ + export excel + :param status: + :param path: int, string + :return: None + """ + results = get_results(status) + if not results: + print("无issue记录") + return False + f = xlwt.Workbook() + sheet1 = f.add_sheet('sheet1', cell_overwrite_ok=True) + # Column field + column_names = ['id', 'cve_id', 'cve_num', 'cve_desc', 'cve_level', + 'cve_version', 'repair_time', 'pack_name', + 'nvd_score', 'n_vector_value', 'create_time', 'update_time', + 'delete_time', 'status', 'error_description'] + # Write the first row, column name + for i in range(0, len(column_names)): + sheet1.write(0, i, column_names[i]) + # Write multiple lines + num = 0 + for i in results: + sheet1.write(num + 1, 0, i["id"]) + sheet1.write(num + 1, 1, i["cve_id"]) + sheet1.write(num + 1, 2, i["cve_num"]) + sheet1.write(num + 1, 3, i["cve_desc"]) + sheet1.write(num + 1, 4, i["cve_level"]) + sheet1.write(num + 1, 5, i["cve_version"]) + sheet1.write(num + 1, 6, i["repair_time"]) + sheet1.write(num + 1, 7, i["pack_name"]) + sheet1.write(num + 1, 8, i["nvd_score"]) + sheet1.write(num + 1, 9, i["n_vector_value"]) + sheet1.write(num + 1, 10, i['create_time']) + sheet1.write(num + 1, 11, i["update_time"]) + sheet1.write(num + 1, 12, i["delete_time"]) + sheet1.write(num + 1, 13, i['status']) + sheet1.write(num + 1, 14, i['error_description']) + num += 1 + # save document + f.save(path + '_' + times.last_month_date()+'_' + cur_date() + '.xls') + update_status(status) diff --git a/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py b/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..c9062546728df299273934743398520ae43d18dc --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py @@ -0,0 +1,286 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle failed tasks + +Authors: zhangjianjun +Date: 04/01/2021 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +import pymysql +from common import files +from common import times +from tabletask import common +from downloadtask import downloadfiletask + + +def download_gauss_yaml(): + """ + Download the yaml file of opengauss on gitee + return file_name + """ + file_path = "./gauss_yaml/" + mk_ok = files.mkdirs(file_path, 1) + file_name = file_path + 'gauss_yaml.yaml' + if mk_ok: + # download the yaml file + file_url = 'https://gitee.com/opengauss/openGauss-third_party' \ + '/raw/master/Third_Party_Open_Source_Software_List.yaml' + # file_url = 'https://gitee.com/buter/openGauss-third_party' \ + # '/raw/master/Third_Party_Open_Source_Software_List.yaml' + downloadfiletask.download_yaml(file_name, file_url) + return file_name + + +def add_package_id(pk_id): + """ + add index + """ + packg_id = 10000000 + if pk_id > 0: + packg_id = pk_id + 1 + return packg_id + + +def store_yaml_data(yaml_data): + """ + parse dict + Store yaml data to mysql + return None + """ + mysql = Mysql() + if yaml_data is not None and len(yaml_data) > 0: + # Delete all data before updating + update_yaml_origin_mark(mysql) + for yaml_key, yaml_value in yaml_data.items(): + try: + if "version" not in yaml_value or not yaml_value["version"]: + yaml_value["version"] = "" + origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql) + if origin_data: + update_yaml_origin_data(origin_data["id"], yaml_value, mysql) + else: + insert_yaml_origin_data(yaml_key, yaml_value, mysql) + packages_data = select_yaml_data(yaml_key, yaml_value, mysql) + if packages_data is not None: + if len(packages_data) > 1: + for pd in packages_data[1:]: + delete_yaml_openeuler_detail_data(mysql, pd["git_id"]) + delete_yaml_openeuler_data(mysql, pd["git_id"]) + pk = packages_data[0] + update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) + update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) + else: + package_ids = select_yaml_lastdata() + if package_ids is not None and package_ids["package_id"] >= 10000000: + packg_id = add_package_id(package_ids["package_id"]) + else: + packg_id = add_package_id(0) + last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) + insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) + mysql.dispose() + except pymysql.err.IntegrityError: + print(pymysql.err.IntegrityError) + # except Exception as e: + # print(e) + mysql.dispose(2) + delete_yaml_origin_mark(mysql) + mysql.close() + + +def select_openeuler_yaml_data(mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler " \ + "WHERE package_id >= %s" + val = (10000000,) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def delete_yaml_openeuler_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_open_euler where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def delete_yaml_openeuler_detail_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_package_info where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def insert_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + insert data + """ + insert_sql = "insert into cve_git_open_euler(package_id,package_name," \ + "version,origin_url,create_time,update_time,cpe_packname,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (package_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), yaml_value["cpeName"], 1) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def insert_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + insert data details + """ + insert_sql = "insert into cve_git_package_info(git_id,package_name," \ + "version,origin_url,create_time,update_time,decription,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (pg_detail_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), "", 0) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def update_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + update data + """ + update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ + "cpe_packname = %s,status=%s where package_id = %s " \ + "and package_name =%s and version = %s" + val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], + 1, package_id, yaml_key, yaml_value["version"]) + mysql.update(update_sql, val) + + +def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + Update data details + """ + update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ + "origin_url = %s, update_time = %s,status=%s where git_id = %s" + val = (yaml_key, yaml_value["version"], yaml_value["url"], + times.get_current_time(), 0, pg_detail_id) + mysql.update(update_sql, val) + + +def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): + """ + Update origin data + """ + update_sql = "update cve_open_guss_yaml set origin_url = %s,status = %s, " \ + "cpe_name = %s, update_time = %s,repo_name=%s, mark_bit = %s where id = %s" + val = (yaml_value["url"], 1, yaml_value["cpeName"], + times.get_current_time(), "security", 1, pg_origin_id) + mysql.update(update_sql, val) + mysql.dispose() + + +def insert_yaml_origin_data(yaml_key, yaml_value, mysql): + """ + insert origin data + """ + insert_sql = "insert into cve_open_guss_yaml(package_name," \ + "version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" + val = (yaml_key, yaml_value["version"], + yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), + times.get_current_time(), 1, "security") + last_id = mysql.insertOne(insert_sql, val) + mysql.dispose() + return last_id + + +def select_yaml_data(yaml_key, yaml_value, mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id FROM cve_git_open_euler " \ + "WHERE package_name = %s " \ + "and version = %s order by package_id desc" + val = (yaml_key, yaml_value["version"]) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def update_yaml_origin_mark(mysql): + """ + Update origin data + """ + update_sql = "update cve_open_guss_yaml set mark_bit = 2 where mark_bit = 1" + mysql.update(update_sql) + mysql.dispose() + + +def delete_yaml_origin_mark(mysql): + """ + delete origin data + """ + delete_sql = "delete from cve_open_guss_yaml where mark_bit = 2" + mysql.delete(delete_sql) + mysql.dispose() + + +def select_yaml_origin_data(package_name, version, mysql): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_guss_yaml WHERE package_name = %s " \ + "and version = %s" + val = (package_name, version) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def select_yaml_lastdata(): + """ + Query the last data + """ + mysql = Mysql() + sql = "SELECT package_id FROM cve_git_open_euler order by git_id desc" + package_ids = mysql.getOne(sql) + if package_ids and len(package_ids) > 0: + return package_ids + else: + return None + + +def proc_gauss_yaml(): + """ + 1. Create a folder; + 2. Download yaml; + 3. Parse yaml; + 4. Save in mysql; + return None + """ + file_name = download_gauss_yaml() + if files.file_isexists(file_name): + yaml_data = files.parse_yaml(file_name) + if yaml_data is not None and len(yaml_data) > 0: + store_yaml_data(yaml_data) + else: + print("File download failed, file: ", file_name) + # delete history yaml_data + common.delete_yaml_data() diff --git a/cve-vulner-manager/cve-py/tabletask/import_excel_task.py b/cve-vulner-manager/cve-py/tabletask/import_excel_task.py new file mode 100644 index 0000000000000000000000000000000000000000..35cb95883a8e6e91afce5e8bc8dfc6423b491e25 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/import_excel_task.py @@ -0,0 +1,117 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +import data from excel + +Authors: xiaojianghui +Date: 11/06/2020 11:01 AM +""" + +import xlrd +import time +import os +import hashlib +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask + + +def parse_excel(file_name): + """ + parse excel, import data + parse: file_name + return: list + """ + pack_name_list = [] + version_list = [] + branchs_list = [] + status_list = [] + total_list = [] + try: + data = xlrd.open_workbook('./import_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + pack_name_list.append(table_one.cell(i, 0).value) + for i in range(1, row_number): + version_list.append(table_one.cell(i, 1).value) + for i in range(1, row_number): + status_list.append(table_one.cell(i, 2).value) + for i in range(1, row_number): + branchs_list.append(table_one.cell(i, 3).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + total_list.append(pack_name_list) + total_list.append(version_list) + total_list.append(status_list) + total_list.append(branchs_list) + return total_list + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def import_data(): + """ + import excel + :param + :return none + """ + downloadfiletask.handle_three() + mysql = Mysql() + files = os.listdir('./import_excels') + for filename in files: + with open('./import_excels/' + filename, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("文件已解析:" + filename) + os.remove('./import_excels/' + filename) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + result = parse_excel(filename) + for i in range(0, len(result[0])): + # sql = 'select * from cve_issue_repo_whitelist where package_name = %s and version = %s' + # val = (result[0][i], result[1][i]) + sql = 'select * from cve_issue_repo_whitelist where package_name = %s' + val = (result[0][i]) + mysql_data = mysql.getOne(sql, val) + if mysql_data: + print("更新数据package_name:" + result[0][i]) + sql = 'update cve_issue_repo_whitelist set status = %s, branchs = %s, update_time = %s where ' \ + 'package_name = %s and version = %s' + val = (result[2][i], result[3][i], cur_date(), result[0][i], result[1][i]) + mysql.update(sql, val) + mysql.dispose() + else: + print('插入数据') + sql = 'insert into cve_issue_repo_whitelist (package_name, version, status, branchs, create_time, ' \ + 'update_time, delete_time) values (%s, %s, %s, %s, %s, %s, %s)' + val = (result[0][i], result[1][i], result[2][i], result[3][i], cur_date(), None, None) + mysql.insertOne(sql, val) + mysql.dispose() + os.remove('./import_excels/' + filename) + mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/issue_statistics.py b/cve-vulner-manager/cve-py/tabletask/issue_statistics.py new file mode 100644 index 0000000000000000000000000000000000000000..15d13225acf193942be9b3b72862757de4b7e321 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/issue_statistics.py @@ -0,0 +1,125 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" + +Authors: zhangjianjun +Date: 01/18/2021 11:01 AM +""" + +import xlrd +import time +import os +import hashlib +import pymysql +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask + + +def proc_excel_data(file_name): + """ + parse excel, import data + parse: file_name + return: list + """ + to_email_name_list = [] + cc_email_name_list = [] + email_dict = {} + try: + data = xlrd.open_workbook(file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + if table_one.cell(i, 1).value == 1: + to_email_name_list.append(table_one.cell(i, 0).value) + else: + cc_email_name_list.append(table_one.cell(i, 0).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + email_dict["to"] = to_email_name_list + email_dict["cc"] = cc_email_name_list + return email_dict + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def get_issue_excel(): + """ + import excel + :param + :return none + """ + file_dir = "issue_statistics" + downloadfiletask.download_excel(file_dir) + mysql = Mysql() + files = os.listdir('./' + file_dir) + for filename in files: + file_vaule = "./%s/%s" % (file_dir, filename) + with open(file_vaule, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("File parsed:" + filename) + os.remove(file_vaule) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + email_dict = proc_excel_data(file_vaule) + if email_dict is not None and len(email_dict) > 0: + to_email_name_list = email_dict["to"] + cc_email_name_list = email_dict["cc"] + else: + continue + if to_email_name_list is not None and len(to_email_name_list) > 0: + del_sql = "delete from cve_issue_statistics_mail_list where email_type = %s" + mysql.delete(del_sql, (1,)) + mysql.dispose() + for email_name in to_email_name_list: + try: + insert_sql = "insert into cve_issue_statistics_mail_list" \ + "(email_name, email_type, create_time) values(%s,%s, %s)" + val = (email_name, 1, cur_date()) + mysql.insertOne(insert_sql, val) + mysql.dispose() + except pymysql.err.IntegrityError as e: + print(e) + mysql.dispose(2) + del_sql = "delete from cve_issue_statistics_mail_list where email_type = %s" + mysql.delete(del_sql, (2,)) + mysql.dispose() + if cc_email_name_list is not None and len(cc_email_name_list) > 0: + for email_name in cc_email_name_list: + try: + insert_sql = "insert into cve_issue_statistics_mail_list" \ + "(email_name, email_type, create_time) values(%s,%s, %s)" + val = (email_name, 2, cur_date()) + mysql.insertOne(insert_sql, val) + mysql.dispose() + except pymysql.err.IntegrityError as e: + print(e) + mysql.dispose(2) + os.remove(file_vaule) + mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/mappingtask.py b/cve-vulner-manager/cve-py/tabletask/mappingtask.py new file mode 100644 index 0000000000000000000000000000000000000000..fa71403ad15cd38e2840c67f6051482e55df135e --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/mappingtask.py @@ -0,0 +1,110 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data + +Authors:xiaojianghui +Date: 10/22/2020 11:01 AM +""" +import hashlib +import xlrd +import os +import time +import shutil +from dbConnecttion import MysqlConn +from downloadtask import downloadfiletask + + +def parse_excel(filename): + """ + parase excel + :param filename: excel file + :return pack_name, cpe_pack_name: string, string + """ + pack_name = [] + cpe_pack_name = [] + try: + data = xlrd.open_workbook("./mappingexcels/" + filename) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + pack_name.append(table_one.cell(i, 0).value) + cpe_pack_name.append(table_one.cell(i, 1).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return pack_name, cpe_pack_name + + +def to_mysql(): + """ + Data stored in the database + """ + downloadfiletask.handle_two() + files = os.listdir("./mappingexcels") + if not files: + print("No package name mapping table") + return False + mysql = MysqlConn.Mysql() + for filename in files: + with open('./mappingexcels/' + filename, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("文件已解析:" + filename) + f_src = os.path.join("./mappingexcels/" + filename) + if not os.path.isdir("./oldexcels/"): + os.mkdir("./oldexcels/") + f_dst = os.path.join("./oldexcels/" + filename) + shutil.move(f_src, f_dst) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + pack_name_list = parse_excel(filename)[0] + cpe_pack_name_list = parse_excel(filename)[1] + if pack_name_list and cpe_pack_name_list: + for j in range(0, len(pack_name_list)): + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + # Query whether data exists in the database, update the data if it exists and is different + sql = "select * from cve_package_cpe where packname = %s" + val = (pack_name_list[j],) + result = mysql.getOne(sql, val) + if result: + # If the query is not empty, execute the next judgment, if it is empty, insert the data + if str(result['packname']) == pack_name_list[j]: + if str(result["cpe_packname"]) != cpe_pack_name_list[j]: + print("update data:" + pack_name_list[j], cpe_pack_name_list[j]) + sql = "update cve_package_cpe set cpe_packname=%s, create_time=%s where packname=%s" + val = (cpe_pack_name_list[j], create_time, pack_name_list[j]) + mysql.update(sql, val) + mysql.dispose() + else: + print("the database is up to date+" + pack_name_list[j], cpe_pack_name_list[j]) + else: + print("insert data:" + pack_name_list[j], cpe_pack_name_list[j]) + sql = "insert into cve_package_cpe (packname,cpe_packname,create_time) values (%s,%s,%s)" + val = (pack_name_list[j], cpe_pack_name_list[j], create_time) + mysql.insertOne(sql, val) + mysql.dispose() + f_src = os.path.join("./mappingexcels/" + filename) + if not os.path.isdir("./oldexcels/"): + os.mkdir("./oldexcels/") + f_dst = os.path.join("./oldexcels/" + filename) + shutil.move(f_src, f_dst) + mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py b/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..0118ebd3f8305e5d34081565493ae95e1924fbf0 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py @@ -0,0 +1,286 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle mindspore tasks + +Authors: zhangjianjun +Date: 05/08/2021 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +import pymysql +from common import files +from common import times +from downloadtask import downloadfiletask +from tabletask import common + + +def download_mindspore_yaml(): + """ + Download the yaml file of mindspore on gitee + return file_name + """ + file_path = "./mindspore_yaml/" + mk_ok = files.mkdirs(file_path, 1) + file_name = file_path + 'mindspore_yaml.yaml' + if mk_ok: + # download the yaml file + file_url = 'https://gitee.com/mindspore/community/raw/' \ + 'master/security/config/Third_Party_Open_Source_Software_List.yaml' + downloadfiletask.download_yaml(file_name, file_url) + # pass + return file_name + + +def add_package_id(pk_id): + """ + add index + """ + packg_id = 10000000 + if pk_id > 0: + packg_id = pk_id + 1 + return packg_id + + +def select_openeuler_yaml_data(mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler " \ + "WHERE package_id >= %s" + val = (10000000,) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def delete_yaml_openeuler_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_open_euler where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def delete_yaml_openeuler_detail_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_package_info where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def insert_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + insert data + """ + insert_sql = "insert into cve_git_open_euler(package_id,package_name," \ + "version,origin_url,create_time,update_time,cpe_packname,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (package_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), yaml_value["cpeName"], 1) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def insert_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + insert data details + """ + insert_sql = "insert into cve_git_package_info(git_id,package_name," \ + "version,origin_url,create_time,update_time,decription,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (pg_detail_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), "", 0) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def update_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + update data + """ + update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ + "cpe_packname = %s,status=%s where package_id = %s " \ + "and package_name =%s and version = %s" + val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], + 1, package_id, yaml_key, yaml_value["version"]) + mysql.update(update_sql, val) + + +def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + Update data details + """ + update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ + "origin_url = %s, update_time = %s,status=%s where git_id = %s" + val = (yaml_key, yaml_value["version"], yaml_value["url"], + times.get_current_time(), 0, pg_detail_id) + mysql.update(update_sql, val) + + +def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): + """ + Update origin data + """ + update_sql = "update cve_mind_spore_yaml set origin_url = %s,status = %s, " \ + "cpe_name = %s, update_time = %s, mark_bit = %s where id = %s" + val = (yaml_value["url"], 1, yaml_value["cpeName"], + times.get_current_time(), 1, pg_origin_id) + mysql.update(update_sql, val) + mysql.dispose() + + +def insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key): + """ + insert origin data + """ + insert_sql = "insert into cve_mind_spore_yaml(package_name," \ + "version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" + val = (yaml_key, yaml_value["version"], + yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), + times.get_current_time(), 1, repo_key) + last_id = mysql.insertOne(insert_sql, val) + mysql.dispose() + return last_id + + +def select_yaml_data(yaml_key, yaml_value, mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id FROM cve_git_open_euler " \ + "WHERE package_name = %s " \ + "and version = %s order by package_id desc" + val = (yaml_key, yaml_value["version"]) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def update_yaml_origin_mark(mysql): + """ + Update origin data + """ + update_sql = "update cve_mind_spore_yaml set mark_bit = 2 where mark_bit = 1" + mysql.update(update_sql) + mysql.dispose() + + +def delete_yaml_origin_mark(mysql): + """ + delete origin data + """ + delete_sql = "delete from cve_mind_spore_yaml where mark_bit = 2" + mysql.delete(delete_sql) + mysql.dispose() + + +def select_yaml_origin_data(package_name, version, mysql, repo_key): + """ + Query origin data + """ + sql = "SELECT id FROM cve_mind_spore_yaml WHERE package_name = %s " \ + "and version = %s and repo_name = %s" + val = (package_name, version, repo_key) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def select_yaml_lastdata(): + """ + Query the last data + """ + mysql = Mysql() + sql = "SELECT package_id FROM cve_git_open_euler order by git_id desc" + package_ids = mysql.getOne(sql) + if package_ids and len(package_ids) > 0: + return package_ids + else: + return None + + +def store_yaml_data(yaml_data): + """ + parse dict + Store yaml data to mysql + return None + """ + mysql = Mysql() + if yaml_data is not None and len(yaml_data) > 0: + # Delete all data before updating + update_yaml_origin_mark(mysql) + for repo_key, repo_value in yaml_data.items(): + try: + for yaml_key, yaml_value in repo_value.items(): + if "version" not in yaml_value or not yaml_value["version"]: + yaml_value["version"] = "" + origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql, repo_key) + if origin_data: + update_yaml_origin_data(origin_data["id"], yaml_value, mysql) + else: + insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key) + packages_data = select_yaml_data(yaml_key, yaml_value, mysql) + if packages_data is not None: + if len(packages_data) > 1: + for pd in packages_data[1:]: + delete_yaml_openeuler_detail_data(mysql, pd["git_id"]) + delete_yaml_openeuler_data(mysql, pd["git_id"]) + pk = packages_data[0] + update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) + update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) + else: + package_ids = select_yaml_lastdata() + if package_ids is not None and package_ids["package_id"] >= 10000000: + packg_id = add_package_id(package_ids["package_id"]) + else: + packg_id = add_package_id(0) + last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) + insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) + mysql.dispose() + except pymysql.err.IntegrityError: + print(pymysql.err.IntegrityError) + # except Exception as e: + # print(e) + mysql.dispose(2) + delete_yaml_origin_mark(mysql) + mysql.close() + + +def proc_mindspore_yaml(): + """ + 1. Create a folder; + 2. Download yaml; + 3. Parse yaml; + 4. Save in mysql; + return None + """ + file_name = download_mindspore_yaml() + if files.file_isexists(file_name): + yaml_data = files.parse_yaml(file_name) + if yaml_data is not None and len(yaml_data) > 0: + store_yaml_data(yaml_data) + else: + print("File download failed, file: ", file_name) + # delete history yaml_data + common.delete_yaml_data() diff --git a/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py b/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..a4f5c95a89ee6d224059258a959febec2bd80605 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py @@ -0,0 +1,286 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle openLooKeng tasks + +Authors: zhangjianjun +Date: 08/25/2021 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +import pymysql +from common import files +from common import times +from downloadtask import downloadfiletask +from tabletask import common + + +def download_openlookeng_yaml(): + """ + Download the yaml file of mindspore on gitee + return file_name + """ + file_path = "./openlookeng_yaml/" + mk_ok = files.mkdirs(file_path, 1) + file_name = file_path + 'openlookeng_yaml.yaml' + if mk_ok: + # download the yaml file + file_url = 'https://gitee.com/openlookeng/community/raw/master/security' \ + '/config/Third_Party_Open_Source_Software_List.yaml' + downloadfiletask.download_yaml(file_name, file_url) + # pass + return file_name + + +def add_package_id(pk_id): + """ + add index + """ + packg_id = 10000000 + if pk_id > 0: + packg_id = pk_id + 1 + return packg_id + + +def select_openeuler_yaml_data(mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler " \ + "WHERE package_id >= %s" + val = (10000000,) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def delete_yaml_openeuler_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_open_euler where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def delete_yaml_openeuler_detail_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_package_info where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def insert_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + insert data + """ + insert_sql = "insert into cve_git_open_euler(package_id,package_name," \ + "version,origin_url,create_time,update_time,cpe_packname,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (package_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), yaml_value["cpeName"], 1) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def insert_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + insert data details + """ + insert_sql = "insert into cve_git_package_info(git_id,package_name," \ + "version,origin_url,create_time,update_time,decription,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (pg_detail_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), "", 0) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def update_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + update data + """ + update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ + "cpe_packname = %s,status=%s where package_id = %s " \ + "and package_name =%s and version = %s" + val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], + 1, package_id, yaml_key, yaml_value["version"]) + mysql.update(update_sql, val) + + +def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + Update data details + """ + update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ + "origin_url = %s, update_time = %s,status=%s where git_id = %s" + val = (yaml_key, yaml_value["version"], yaml_value["url"], + times.get_current_time(), 0, pg_detail_id) + mysql.update(update_sql, val) + + +def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): + """ + Update origin data + """ + update_sql = "update cve_open_lookeng_yaml set origin_url = %s,status = %s, " \ + "cpe_name = %s, update_time = %s,mark_bit = %s where id = %s" + val = (yaml_value["url"], 1, yaml_value["cpeName"], + times.get_current_time(), 1, pg_origin_id) + mysql.update(update_sql, val) + mysql.dispose() + + +def insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key): + """ + insert origin data + """ + insert_sql = "insert into cve_open_lookeng_yaml(package_name," \ + "version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" + val = (yaml_key, yaml_value["version"], + yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), + times.get_current_time(), 1, repo_key) + last_id = mysql.insertOne(insert_sql, val) + mysql.dispose() + return last_id + + +def select_yaml_data(yaml_key, yaml_value, mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id FROM cve_git_open_euler " \ + "WHERE package_name = %s " \ + "and version = %s order by package_id desc" + val = (yaml_key, yaml_value["version"]) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def update_yaml_origin_mark(mysql): + """ + Update origin data + """ + update_sql = "update cve_open_lookeng_yaml set mark_bit = 2 where mark_bit = 1" + mysql.update(update_sql) + mysql.dispose() + + +def delete_yaml_origin_mark(mysql): + """ + delete origin data + """ + delete_sql = "delete from cve_open_lookeng_yaml where mark_bit = 2" + mysql.update(delete_sql) + mysql.dispose() + + +def select_yaml_origin_data(package_name, version, mysql, repo_key): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = %s " \ + "and version = %s and repo_name = %s" + val = (package_name, version, repo_key) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def select_yaml_lastdata(): + """ + Query the last data + """ + mysql = Mysql() + sql = "SELECT package_id FROM cve_git_open_euler order by git_id desc" + package_ids = mysql.getOne(sql) + if package_ids and len(package_ids) > 0: + return package_ids + else: + return None + + +def store_yaml_data(yaml_data): + """ + parse dict + Store yaml data to mysql + return None + """ + mysql = Mysql() + if yaml_data is not None and len(yaml_data) > 0: + # Delete all data before updating + update_yaml_origin_mark(mysql) + for repo_key, repo_value in yaml_data.items(): + try: + for yaml_key, yaml_value in repo_value.items(): + if "version" not in yaml_value or not yaml_value["version"]: + yaml_value["version"] = "" + origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql, repo_key) + if origin_data: + update_yaml_origin_data(origin_data["id"], yaml_value, mysql) + else: + insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key) + packages_data = select_yaml_data(yaml_key, yaml_value, mysql) + if packages_data is not None: + if len(packages_data) > 1: + for pd in packages_data[1:]: + delete_yaml_openeuler_detail_data(mysql, pd["git_id"]) + delete_yaml_openeuler_data(mysql, pd["git_id"]) + pk = packages_data[0] + update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) + update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) + else: + package_ids = select_yaml_lastdata() + if package_ids is not None and package_ids["package_id"] >= 10000000: + packg_id = add_package_id(package_ids["package_id"]) + else: + packg_id = add_package_id(0) + last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) + insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) + mysql.dispose() + except pymysql.err.IntegrityError: + print(pymysql.err.IntegrityError) + # except Exception as e: + # print(e) + mysql.dispose(2) + delete_yaml_origin_mark(mysql) + mysql.close() + + +def proc_openlookeng_yaml(): + """ + 1. Create a folder; + 2. Download yaml; + 3. Parse yaml; + 4. Save in mysql; + return None + """ + file_name = download_openlookeng_yaml() + if files.file_isexists(file_name): + yaml_data = files.parse_yaml(file_name) + if yaml_data is not None and len(yaml_data) > 0: + store_yaml_data(yaml_data) + else: + print("File download failed, file: ", file_name) + # delete history yaml_data + common.delete_yaml_data() diff --git a/cve-vulner-manager/cve-py/tabletask/package_committer_task.py b/cve-vulner-manager/cve-py/tabletask/package_committer_task.py new file mode 100644 index 0000000000000000000000000000000000000000..48b240100611906161dc92617dc0237bc1b45e1c --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/package_committer_task.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +parse excel file and mysql data + +Authors:xiaojianghui +Date: 11/27/2020 11:01 AM +""" + +import os +import xlrd +import time +import hashlib +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask + + +def parse_excel(file_name): + """ + parse excel data + parse: file_name + return: list + """ + package_name_list = [] + issue_assignee_list = [] + try: + data = xlrd.open_workbook('./package_committer_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + package_name_list.append(table_one.cell(i, 0).value) + issue_assignee_list.append(table_one.cell(i, 1).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return package_name_list, issue_assignee_list + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def import_data(): + """ + import excel + :param + :return none + """ + downloadfiletask.handle_four() + mysql = Mysql() + files = os.listdir('./package_committer_excels') + for filename in files: + with open('./package_committer_excels/' + filename, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("文件已解析:" + filename) + os.remove('./package_committer_excels/' + filename) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + result = parse_excel(filename) + for i in range(0, len(result[0])): + sql = 'insert ignore into cve_spec_issue_assigness (package_name, issue_assignee, status, create_time) ' \ + 'values (%s, %s, %s, %s)' + val = (result[0][i], result[1][i], 1, cur_date()) + mysql.insertOne(sql, val) + mysql.dispose() + print('插入数据{}:{}'.format(result[0][i], result[1][i])) + os.remove('./package_committer_excels/' + filename) + mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/repeattask.py b/cve-vulner-manager/cve-py/tabletask/repeattask.py new file mode 100644 index 0000000000000000000000000000000000000000..c0bce3d61dae49e8a6e625f00d510d95c65b1ba4 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/repeattask.py @@ -0,0 +1,84 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle failed tasks + +Authors: xiaojianghui +Date: 21/12/2020 11:01 AM +""" + +import time +from dbConnecttion.MysqlConn import Mysql +from tabletask import crawltask + + +def cur_date(): + """ + current date + Returns:createTime:string + + """ + create_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + return create_time + + +def repeat_task(): + """ + Parse the artificial CVE table, crawl the CVE official + website data, and store it in the database + return None + """ + print("CVE官网数据为空的CVE进行再次抓取") + mysql = Mysql() + sql = "SELECT cve_num,pack_name,cve_version " \ + "FROM cve_origin_excel WHERE score_type = %s " \ + "OR score_type IS NULL OR cve_desc IS NULL OR nvd_score IS NULL" + val = ('',) + result = mysql.getMany(sql, val) + print("CVE官网数据为空的 数据：", result) + for i in result: + crawllist = crawltask.crawling("https://nvd.nist.gov/vuln/detail/" + i["cve_num"]) + print("更新CVE数据") + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ + "score_type=%s where cve_num=%s and pack_name = %s and cve_version = %s" + val = (crawllist[0], crawllist[1], crawllist[2], crawllist[3], + crawllist[4], crawllist[5], crawllist[6], + crawllist[7], crawllist[8], crawllist[9], crawllist[10], + crawllist[11], crawllist[12], crawllist[13], + crawllist[14], crawllist[15], 1, cur_date(), crawllist[16], + i["cve_num"], i["pack_name"], i["cve_version"]) + mysql.update(sql, val) + mysql.dispose() + mysql.close() + + +def get_published_date_task(): + """ + Get the release date of nvd + return None + """ + print("Get the release date of nvd") + mysql = Mysql() + sql = "SELECT cve_id, cve_num, repair_time " \ + "FROM cve_vuln_center WHERE repair_time = '' " \ + "OR repair_time IS NULL" + result = mysql.getMany(sql) + if result is not None and result[0] is not None: + for i in result: + crawllist = crawltask.crawling("https://nvd.nist.gov/vuln/detail/" + i["cve_num"]) + if crawllist is not None and len(crawllist) >= 4 and crawllist[3]: + sql = "update cve_vuln_center set repair_time = %s where cve_id = %s" + val = (crawllist[3], i["cve_id"]) + mysql.update(sql, val) + mysql.dispose() + mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/runtask.py b/cve-vulner-manager/cve-py/tabletask/runtask.py new file mode 100644 index 0000000000000000000000000000000000000000..b926cf02702358aee7cbb752d498f0fd450a0b1c --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/runtask.py @@ -0,0 +1,166 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +from tabletask import exceltask, crawltask +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask +import time +import os +import hashlib + + +def handle_data(): + """ + CVSS official website data crawling data storage database + """ + # downloadfiletask.handle_one() + path = "./newexcels" + if not os.path.isdir(path): + print("There is currently no excel data executable") + return + files = os.listdir(path) + if files: + mysql = Mysql() + for fileName in files: + with open('./newexcels/' + fileName, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(fileName, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (fileName,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("文件已解析:" + fileName) + exceltask.move_file(fileName) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (fileName, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + print("File name currently being processed: ", fileName) + cve_num_list = exceltask.crawl_cve_num(fileName) + urls = exceltask.crawl_urls(fileName) + cve_version_list = exceltask.crawl_cve_version(fileName) + pack_name_list = exceltask.crawl_packname(fileName) + for i in range(0, len(urls)): + cve_num = str(cve_num_list[i]).strip() + cve_version = str(cve_version_list[i]) + pack_name = str(pack_name_list[i]) + print(fileName, cve_num, pack_name, cve_version) + # Database query results + sql = "select * from cve_origin_excel where " \ + "cve_num= %s and pack_name = %s and cve_version = %s" + val = (cve_num, pack_name, cve_version) + result_dict = mysql.getOne(sql, val) + # Determine whether CVE exists in the database + if result_dict: + # Crawler web data + crawl_list = crawltask.crawling(urls[i]) + print(crawl_list) + # Determine whether the database content is the latest data + if crawl_list[0] or crawl_list[2]: + if str(result_dict["nvd_score"]) == str(crawl_list[0]) \ + and str(result_dict["vector_value"]) == str(crawl_list[4]): + if result_dict['cve_status'] in [3, 4, 7]: + print("update data:" + cve_num) + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + try: + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" + val = ( + crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], + crawl_list[5], + crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], + crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], + crawl_list[15], 1, update_time, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + except IndexError as e: + mysql.dispose(0) + print("Subscript out of bounds", e) + else: + print("The database is the latest data:" + cve_num) + else: + print("update data:" + cve_num) + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + try: + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ + "score_type=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" + val = ( + crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], + crawl_list[5], crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], + crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], + crawl_list[15], 1, update_time, crawl_list[16], + cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + except IndexError as e: + print("Subscript out of bounds", e) + mysql.dispose(0) + else: + print("error: ", result_dict) + else: + print("insert data") + create_time = update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + delete_time = None + # State 0 means new, 1 means modified + cve_status = 0 + try: + cve_url = str(urls[i]) + listx = crawltask.crawling(cve_url) + sql = "INSERT INTO cve_origin_excel (cve_num, cve_url, cve_version, pack_name, score_type, " \ + "nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, " \ + "access_vector, attack_complexity, access_complexity, privilege_required, " \ + "user_interaction, scope, confidentiality, integrity, availability, " \ + "authentication, cve_status, " \ + "create_time, update_time, delete_time) " \ + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \ + "%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + val = ( + cve_num, cve_url, cve_version, pack_name, listx[16], listx[0], listx[1], listx[2], listx[3], + listx[4], + listx[5], listx[6], listx[7], listx[8], listx[9], listx[10], listx[11], listx[12], + listx[13], + listx[14], listx[15], cve_status, create_time, update_time, delete_time) + mysql.insertOne(sql, val) + mysql.dispose() + except IndexError as e: + print("Subscript out of bounds", e) + mysql.dispose(0) + sql = "select * from cve_spec_error where cve_num = %s" + val = (cve_num,) + result_spec_error = mysql.getOne(sql, val) + if result_spec_error: + print("过滤,修改status为6：{}".format(cve_num)) + sql = "update cve_origin_excel set cve_desc = %s, cve_status = %s where cve_num = %s" + val = (result_spec_error["cve_desc"], 6, cve_num) + mysql.update(sql, val) + mysql.dispose() + exceltask.move_file(fileName) + mysql.close() + else: + print("error: There are no manually added tables in the newexcels folder") diff --git a/cve-vulner-manager/cve-py/tabletask/spec_error_task.py b/cve-vulner-manager/cve-py/tabletask/spec_error_task.py new file mode 100644 index 0000000000000000000000000000000000000000..9abea93fe1843d07e23d662fb997ba1f78bc5e5c --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/spec_error_task.py @@ -0,0 +1,123 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data +Authors: xiaojianghui +Date: 12/08/2020 11:01 AM +""" + +import os +import xlrd +import time +import urllib.parse +from sqlalchemy import Column, String, create_engine, Integer, Boolean +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from downloadtask import downloadfiletask +from dbConnecttion import Config + +Base = declarative_base() + + +class Details(Base): + """ + sqlalchemy orm + """ + __tablename__ = 'cve_spec_error' + id = Column(Integer, primary_key=True) + cve_num = Column(String) + cve_desc = Column(String) + cve_owner = Column(String) + cve_status = Column(Boolean) + pack_name = Column(String) + create_time = Column(String) + update_time = Column(String) + delete_time = Column(String) + + +def cur_date(): + """ + current date + Returns:createTime:string + + """ + create_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + return create_time + + +def parse_error_excels(file_name): + """ + parse spec_error_excels files + Returns: + + """ + cve_num_list = [] + cve_desc_list = [] + cve_owner_list = [] + pack_name_list = [] + list_all = [] + try: + data = xlrd.open_workbook('./spec_error_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + if table_one.cell(i, 1).value: + cve_num_list.append(table_one.cell(i, 1).value) + else: + continue + if table_one.cell(i, 4).value: + cve_desc_list.append(table_one.cell(i, 4).value) + else: + cve_desc_list.append("") + if table_one.cell(i, 2).value: + cve_owner_list.append(table_one.cell(i, 2).value) + else: + cve_owner_list.append("") + if table_one.cell(i, 3).value: + pack_name_list.append(table_one.cell(i, 3).value) + else: + pack_name_list.append("") + list_all = [cve_num_list, cve_desc_list, cve_owner_list, pack_name_list] + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return list_all + + +def add_error_details(): + """ + Conditions for filtering incorrect data + Returns: + + """ + downloadfiletask.download_spec_error_excels() + files = os.listdir('./spec_error_excels') + for file in files: + result = parse_error_excels(file) + if not result[0]: + continue + print('开始批量插入人工CVE过滤条件') + objects = [] + pwd = urllib.parse.quote_plus(Config.DBPWD) + engine = create_engine( + "mysql://" + Config.DBUSER + ":" + pwd + "@" + Config.DBHOST + "/" + Config.DBNAME + "?charset=utf8") + for i in range(0, len(result[0])): + objects.append(Details(cve_num=result[0][i], cve_desc=result[1][i], cve_owner=result[2][i], + cve_status=1, pack_name=result[3][i], create_time=cur_date())) + db_session = sessionmaker(bind=engine) + session = db_session() + try: + session.bulk_save_objects(objects) + except IntegrityError: + session.rollback() + session.commit() + session.close() + os.remove('./spec_error_excels/' + file) diff --git a/cve-vulner-manager/cve-py/tabletask/supplement_cve.py b/cve-vulner-manager/cve-py/tabletask/supplement_cve.py new file mode 100644 index 0000000000000000000000000000000000000000..5667adc13f43479390b616cf3fab85748fdceccd --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/supplement_cve.py @@ -0,0 +1,372 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2021 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Supplemental cve information + +Authors: zhangjianjun +Date: 1/19/2021 11:01 AM +""" + +from tabletask import crawltask +from dbConnecttion.MysqlConn import Mysql +import time +import datetime + + + +def query_cve_all_data(mysql, days): + """ + Find cve with missing necessary fields + """ + before_date = (datetime.date.today() - + datetime.timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S") + score_sql = "select cve_id,cve_num from cve_score " \ + "where nvd_score = 0 and update_time >= %s order by update_time desc" + val = (before_date,) + cve_list = [] + cve_id_list = [] + score_result = mysql.getMany(score_sql, val) + if score_result and len(score_result) > 0: + for sc in score_result: + if sc and sc not in cve_id_list: + cve_id_list.append(sc) + + cve_desc_sql = "select cve_id,cve_num from " \ + "cve_vuln_center where (cve_desc = %s or " \ + "repair_time = %s) and update_time >= %s " \ + "order by cve_id desc" + center_val = ("", "", before_date) + center_result = mysql.getMany(cve_desc_sql, center_val) + if center_result and len(center_result) > 0: + for ce in center_result: + if ce and ce not in cve_id_list: + cve_id_list.append(ce) + + cve_template_sql = "select cve_id,cve_num from " \ + "cve_issue_template where (nvd_score = %s or " \ + "nvd_vector = %s or cve_brief = %s) and update_time >= %s " \ + "order by cve_id desc" + template_val = (0, "", "", before_date) + template_result = mysql.getMany(cve_template_sql, template_val) + if template_result and len(template_result) > 0: + for ct in template_result: + if ct and ct not in cve_id_list: + cve_id_list.append(ct) + + if cve_id_list and len(cve_id_list) > 0: + for ci in cve_id_list: + center_sql = "select cve_num, pack_name, cve_version," \ + "cve_desc,repair_time,cve_status,cve_id from " \ + "cve_vuln_center where cve_id = %s and " \ + "cve_num = %s order by cve_id desc" + center_val = (ci["cve_id"], ci["cve_num"]) + center_result = mysql.getOne(center_sql, center_val) + if center_result: + cve_list.append(center_result) + return cve_list + + +def update_cve(url, result_dict, cve, mysql): + """ + update data + """ + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + # Crawler web data + crawl_list = crawltask.crawling(url) + print(crawl_list) + # Determine whether the database content is the latest data + if crawl_list[0]: + if str(result_dict["nvd_score"]) == str(crawl_list[0]) \ + and str(result_dict["vector_value"]) == str(crawl_list[4]): + if result_dict['cve_status'] in [3, 4, 7]: + print("update data:" + cve_num) + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + try: + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" + val = ( + crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], + crawl_list[5], + crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], + crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], + crawl_list[15], 1, update_time, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + except IndexError as e: + mysql.dispose(0) + print("Subscript out of bounds", e) + else: + print("The database is the latest data:" + cve_num) + else: + print("update data:" + cve_num) + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + try: + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ + "score_type=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" + val = ( + crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], + crawl_list[5], crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], + crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], + crawl_list[15], 1, update_time, crawl_list[16], + cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + except IndexError as e: + print("Subscript out of bounds", e) + mysql.dispose(0) + else: + print("error: ", result_dict) + + +def update_cve_vuln(url, cve, mysql): + """ + update data + """ + ret_value = 0 + cve_num = str(cve["cve_num"]).strip() + cve_id = cve["cve_id"] + cve_status = cve["cve_status"] + if cve["cve_desc"] is not None: + cve_desc = str(cve["cve_desc"]).strip() + else: + cve_desc = "" + if cve["repair_time"] is not None: + repair_time = str(cve["repair_time"]).strip() + else: + repair_time = "" + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + # State 0 means new, 1 means modified + if cve_status in (0, 3, 4, 5, 6, 7, 8): + cve_statux = 0 + elif cve_status in (1, 2): + cve_statux = 1 + else: + cve_statux = cve_status + try: + listx = crawltask.crawling(url) + if (listx[2] is None or listx[2] == "" or len(listx[2]) < 2) and \ + (listx[0] is None or listx[0] == 0) and \ + (listx[3] is None or listx[3] == "" or len(listx[3]) < 2): + print("The data does not exist and will not be processed temporarily", listx) + return ret_value + if repair_time == "" and listx[3] is not None and len(listx[3]) > 2: + if len(listx[3]) > 10: + repair_times = listx[3][:10] + else: + repair_times = listx[3] + sql = "update cve_vuln_center set repair_time = %s,update_time=%s," \ + "cve_status=%s where cve_id=%s" + val = (repair_times, update_time, cve_statux, cve_id) + mysql.update(sql, val) + mysql.dispose() + if listx[2] is not None and len(listx[2]) > 2 and cve_desc == "": + sql = "update cve_vuln_center set cve_desc = %s,update_time=%s," \ + "cve_status=%s where cve_id=%s" + val = (listx[2], update_time, cve_statux, cve_id) + mysql.update(sql, val) + mysql.dispose() + if listx[0] is not None and float(listx[0]) > 0: + if listx[16] == "v3.0": + score_type = "v3" + else: + score_type = "v2" + sql = "select id,nvd_score from cve_score where cve_id = %s and " \ + "cve_num = %s" + val = (cve_id, cve_num) + nvd_result = mysql.getOne(sql, val) + if nvd_result and len(nvd_result) > 0 and nvd_result["nvd_score"] == 0: + sql = "update cve_score set nvd_score=%s, " \ + "n_vector_value=%s, n_attack_vector=%s, n_access_vector=%s, " \ + "n_attack_complexity=%s, n_access_complexity=%s, n_privilege_required=%s, " \ + "n_user_interaction=%s, n_scope=%s, n_confidentiality=%s, n_integrity=%s, " \ + "n_availability=%s, n_authentication=%s, update_time=%s,score_type=%s " \ + "where id = %s" + val = ( + listx[0], listx[4], + listx[5], listx[6], listx[7], listx[8], listx[9], + listx[10], listx[11], listx[12], listx[13], listx[14], + listx[15], update_time, score_type, nvd_result["id"]) + mysql.update(sql, val) + mysql.dispose() + cve_template_sql = "select template_id,nvd_score,nvd_vector,cve_brief from " \ + "cve_issue_template where (nvd_score = %s or " \ + "nvd_vector = %s or cve_brief = %s) and cve_id = %s and cve_num = %s" + template_val = (0, "", "", cve_id, cve_num) + template_result = mysql.getOne(cve_template_sql, template_val) + if template_result and len(template_result) > 0: + flag = False + if template_result["nvd_score"] == 0 and listx[0] and float(listx[0]) > 0: + temp_update_sql = "update cve_issue_template set nvd_score = %s," \ + "update_time=%s where template_id = %s" + val = (listx[0], update_time, template_result["template_id"]) + mysql.update(temp_update_sql, val) + mysql.dispose() + flag = True + if (not template_result["nvd_vector"] or len(template_result["nvd_vector"]) < 2) \ + and (listx[4] and len(listx[4]) > 2): + temp_update_sql = "update cve_issue_template set nvd_vector = %s," \ + "update_time=%s where template_id = %s" + val = (listx[4], update_time, template_result["template_id"]) + mysql.update(temp_update_sql, val) + mysql.dispose() + flag = True + if (not template_result["cve_brief"] or len(template_result["cve_brief"]) < 2) \ + and (listx[2] and len(listx[2]) > 2): + temp_update_sql = "update cve_issue_template set cve_brief = %s," \ + "update_time=%s where template_id = %s" + val = (listx[2], update_time, template_result["template_id"]) + mysql.update(temp_update_sql, val) + mysql.dispose() + flag = True + if flag: + sql = "update cve_vuln_center set update_time=%s," \ + "cve_status=%s where cve_id=%s" + val = (update_time, cve_statux, cve_id) + mysql.update(sql, val) + mysql.dispose() + ret_value = 1 + except IndexError as e: + print("Subscript out of bounds", e) + mysql.dispose(0) + return ret_value + + +def supplement_cve(): + """ + 1. Find cve with missing necessary fields; + 2. Go to the CVE official website to find the CVE information; + 3. Fill in the corresponding table again; + return None + """ + cve_list_data = list() + mysql = Mysql() + cve_list = query_cve_all_data(mysql, 3) + if cve_list is not None and len(cve_list) > 0: + for info in cve_list: + if info not in cve_list_data: + cve_list_data.append(info) + if cve_list_data is not None and len(cve_list_data) > 0: + for cve in cve_list_data: + print(cve) + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + url = "https://nvd.nist.gov/vuln/detail/" + cve_num + update_cve_vuln(url, cve, mysql) + sql = "select * from cve_spec_error where cve_num = %s and " \ + "cve_owner = %s and pack_name = %s" + val = (cve_num, "src-openEuler", pack_name) + result_spec_error = mysql.getOne(sql, val) + if result_spec_error: + print("过滤,修改status为6：{}".format(cve_num)) + sql = "update cve_origin_excel set cve_desc = %s, cve_status = %s " \ + "where cve_num= %s and pack_name = %s and cve_version = %s" + val = (result_spec_error["cve_desc"], 6, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + mysql.close() + + +def long_supplement_cve(): + """ + 1. Find cve with missing necessary fields; + 2. Go to the CVE official website to find the CVE information; + 3. Fill in the corresponding table again; + return None + """ + cve_list_data = list() + mysql = Mysql() + cve_list = query_cve_all_data(mysql, 1500) + if cve_list is not None and len(cve_list) > 0: + for info in cve_list: + if info not in cve_list_data: + cve_list_data.append(info) + if cve_list_data is not None and len(cve_list_data) > 0: + for cve in cve_list_data: + print(cve) + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + url = "https://nvd.nist.gov/vuln/detail/" + cve_num + update_cve_vuln(url, cve, mysql) + sql = "select * from cve_spec_error where cve_num = %s and " \ + "cve_owner = %s and pack_name = %s" + val = (cve_num, "src-openEuler", pack_name) + result_spec_error = mysql.getOne(sql, val) + if result_spec_error: + print("过滤,修改status为6：{}".format(cve_num)) + sql = "update cve_origin_excel set cve_desc = %s, cve_status = %s " \ + "where cve_num= %s and pack_name = %s and cve_version = %s" + val = (result_spec_error["cve_desc"], 6, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + mysql.close() + + +def query_cve_by_number(mysql, cve_num): + """ + Find cve with missing necessary fields + """ + center_sql = "select cve_num, pack_name, cve_version," \ + "cve_desc,repair_time,cve_status,cve_id from " \ + "cve_vuln_center where " \ + "cve_num = %s order by cve_id desc" + center_val = (cve_num) + cve_list = mysql.getMany(center_sql, center_val) + return cve_list + + +def pull_cve(cve_num): + """ + 1. Find cve with missing necessary fields; + 2. Go to the CVE official website to find the CVE information; + 3. Fill in the corresponding table again; + return None + """ + cve_list_data = list() + ret_value = 0 + mysql = Mysql() + cve_list = query_cve_by_number(mysql, cve_num) + if cve_list and len(cve_list) > 0: + for info in cve_list: + if info not in cve_list_data: + cve_list_data.append(info) + print(cve_list_data) + if cve_list_data and len(cve_list_data) > 0: + for cve in cve_list_data: + print(cve) + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + url = "https://nvd.nist.gov/vuln/detail/" + cve_num + ret_value = update_cve_vuln(url, cve, mysql) + sql = "select * from cve_spec_error where cve_num = %s and " \ + "cve_owner = %s and pack_name = %s" + val = (cve_num, "src-openEuler", pack_name) + result_spec_error = mysql.getOne(sql, val) + if result_spec_error: + print("过滤,修改status为6：{}".format(cve_num)) + sql = "update cve_origin_excel set cve_desc = %s, cve_status = %s " \ + "where cve_num= %s and pack_name = %s and cve_version = %s" + val = (result_spec_error["cve_desc"], 6, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + mysql.close() + return ret_value diff --git a/cve-vulner-manager/cve-py/tabletask/toexcel.py b/cve-vulner-manager/cve-py/tabletask/toexcel.py new file mode 100644 index 0000000000000000000000000000000000000000..71facfb575dd2c5c5ff32154ef2b3e0b77709c15 --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/toexcel.py @@ -0,0 +1,139 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import os +import xlwt +from dbConnecttion.MysqlConn import Mysql +import time +import shutil +from common import times + + +def cur_date(): + """ + current date + :return create_time: string + """ + create_time = time.strftime("%Y-%m-%d", time.localtime()) + return create_time + + +class MysqlToExcel(object): + """Error data is exported from mysql to excel""" + + def __init__(self): + """Error data is exported from mysql to excel""" + self.file_name = './problemexcels/error_data_' + str(cur_date()) + ".xls" + + @staticmethod + def get_results(): + """Query error data + :return results: list + """ + mysql = Mysql() + last_month = times.last_month_date() + sql = "select * from cve_origin_excel where (cve_status = 3 or cve_status=4) and is_export = 1 and create_time > %s" + val = (last_month,) + results = mysql.getMany(sql, val) + print(results) + mysql.dispose() + mysql.close() + return results + + def generate_table(self): + """Generate excel sheet, File already exists, delete""" + if os.listdir("./problemexcels"): + shutil.rmtree("./problemexcels") + os.mkdir("./problemexcels") + results = self.get_results() + if not results: + print("The query result is empty, no error data") + return + # Create excel object + f = xlwt.Workbook() + sheet1 = f.add_sheet('sheet1', cell_overwrite_ok=True) + # Column field + column_names = ['cve_id', 'cve_num', 'cve_url', 'cve_version', 'pack_name', 'score_type', 'nvd_score', + 'cve_level', 'cve_desc', + 'repair_time', 'vector_value', 'attack_vector', 'access_vector', 'attack_complexity', + 'access_complexity', + 'privilege_required', 'user_interaction', 'scope', 'confidentiality', 'integrity', + 'availability', + 'authentication', 'cve_status', 'create_time', 'update_time', + 'delete_time'] + # Write the first row, column name + for i in range(0, len(column_names)): + sheet1.write(0, i, column_names[i]) + # Write multiple lines + num = 0 + for i in results: + for key in i: + if type(i[key]) == bytes: + i[key] = i[key].decode("utf-8") + sheet1.write(num + 1, 0, i["cve_id"]) + sheet1.write(num + 1, 1, i["cve_num"]) + sheet1.write(num + 1, 2, i["cve_url"]) + sheet1.write(num + 1, 3, i["cve_version"]) + sheet1.write(num + 1, 4, i["pack_name"]) + sheet1.write(num + 1, 5, i["score_type"]) + sheet1.write(num + 1, 6, i["nvd_score"]) + sheet1.write(num + 1, 7, i["cve_level"]) + sheet1.write(num + 1, 8, i["cve_desc"]) + sheet1.write(num + 1, 9, i["repair_time"]) + sheet1.write(num + 1, 10, i["vector_value"]) + sheet1.write(num + 1, 11, i["attack_vector"]) + sheet1.write(num + 1, 12, i["access_vector"]) + sheet1.write(num + 1, 13, i["attack_complexity"]) + sheet1.write(num + 1, 14, i["access_complexity"]) + sheet1.write(num + 1, 15, i["privilege_required"]) + sheet1.write(num + 1, 16, i["user_interaction"]) + sheet1.write(num + 1, 17, i["scope"]) + sheet1.write(num + 1, 18, i["confidentiality"]) + sheet1.write(num + 1, 19, i["integrity"]) + sheet1.write(num + 1, 20, i["availability"]) + sheet1.write(num + 1, 21, i["authentication"]) + sheet1.write(num + 1, 22, i["cve_status"]) + if i['create_time']: + sheet1.write(num + 1, 23, i["create_time"].strftime('%Y-%m-%d %H:%M:%S')) + else: + sheet1.write(num + 1, 23, i['create_time']) + if i["update_time"]: + sheet1.write(num + 1, 24, i["update_time"].strftime('%Y-%m-%d %H:%M:%S')) + else: + sheet1.write(num + 1, 24, i["update_time"]) + if i["delete_time"]: + sheet1.write(num + 1, 25, i["delete_time"].strftime('%Y-%m-%d %H:%M:%S')) + else: + sheet1.write(num + 1, 25, i["delete_time"]) + num += 1 + # save document + f.save(self.file_name) + # Determine whether the file exists + if not os.path.exists(self.file_name): + print("Failed to generate excel") + else: + print("Excel generated successfully") + + @staticmethod + def update_status(): + """ + Change the data with is_export 1 to 2 to indicate that it has been exported in excel + """ + mysql = Mysql() + sql = "update cve_origin_excel set is_export=2 where cve_status=3 or cve_status=4" + mysql.update(sql) + mysql.dispose() + mysql.close() diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go new file mode 100644 index 0000000000000000000000000000000000000000..fe1371e41295eb80e7f39d440fd123b7e7cda59b --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -0,0 +1,122 @@ +package controller + +import ( + "cvevulner/cve-timed-task/deletetask" + "cvevulner/cve-timed-task/emailtask" + "cvevulner/cve-timed-task/gitwebtask" + "cvevulner/cve-timed-task/tabletask" + "cvevulner/cve-timed-task/util" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "time" +) + +// GwController Grab warehouse yaml data +func GwController() { + logs.Info("START TASK:", "Start of crawling organization, warehouse, owners correspondence relationship timing task") + gitwebtask.Handle() + //TODO + logs.Info("END TASK:", "Grab warehouse yaml data task completed") +} + +// RunTableTask Parse the artificial CVE table, crawl the CVE official +//website data, and store it in the database +func RunTableTask() { + logs.Info("START TASK:", "Analyze the manual data table, crawl the CVE official website data, and start the timing task") + tabletask.AddErrorDetails() + tabletask.HandleDate() + tabletask.RepeatTask() + tabletask.GetPublishedDateTask() + logs.Info("END TASK:", "Analyze the manual data table, crawl the CVE official website data task completed") +} + +// ToExcelController Query the data whose cve_status status is 3 or 4 in the +// database to generate excel Modify the status of cve_status to 3 or 4 to 5, +// indicating that it has been added to excel to be processed +func ToExcelController() { + logs.Info("START TASK:", "Export data to generate excel, send error data feedback email task start") + filePath := beego.AppConfig.String("timedtask::packageDir") + "problemexcels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return + } + createTime := time.Now().Format("2006-01-02") + + tabletask.GenerateTable(filePath + "error_data_" + createTime + ".xls") + emailtask.SendingTypeTwo(filePath) + logs.Info("END TASK:", "To excel and send email task completed") +} + +// RunMappingTask Parse the artificial CVE table, crawl the CVE +//official website data, and store it in the database +func RunMappingTask() { + logs.Info("START TASK:", "The timing task of parsing the package name mapping table starts") + tabletask.ToMysql() + logs.Info("END TASK:", "The timing task of parsing the package name mapping table completed") +} + +// RunDelFileTask Delete files older than one month +func RunDelFileTask() { + logs.Info("START TASK:", "Clean up the folder timed task starts") + deletetask.DelDir(beego.AppConfig.String("timedtask::packageDir") + "oldexcels/") + logs.Info("END TASK:", "Clean up the folder task completed") +} + +// RunIssueRecordTask issues record email task +func RunIssueRecordTask() { + logs.Info("START TASK:", "issues record email task start") + filePath := beego.AppConfig.String("timedtask::packageDir") + "export_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.MakeDir error:", err, filePath) + return + } + tabletask.GenerateExcels("1", filePath+"create_issues.xls") + tabletask.GenerateExcels("2", filePath+"unctreated_issues.xls") + emailtask.Sending(filePath) + logs.Info("END TASK:", "issues record email task complete") +} + +// RunWhitelistTask the task of parsing package whitelist +func RunWhitelistTask() { + logs.Info("START TASK:", "the task of parsing package whitelist start") + tabletask.ImportDataTypeTwo() + logs.Info("END TASK:", "the task of parsing package whitelist complete") +} + +// RunPackageCommitterTask the task of parsing package_committer excels +func RunPackageCommitterTask() { + logs.Info("START TASK:", "the task of parsing package_committer_excels start") + tabletask.ImportData() + logs.Info("END TASK:", "the task of parsing package_committer_excels complete") +} + +// SupplementCveTask Complete the template information of the issue with the data on the CVE official website +func SupplementCveTask() { + logs.Info("START TASK:", "Supplemental cve information task starts") + tabletask.SupplementCve() + logs.Info("END TASK:", "Supplement cve information task is over") +} + +// LongSupplementCveTask Complete the template information of the +//issue with the data on the CVE official website +func LongSupplementCveTask() { + logs.Info("START TASK:", "long Supplemental cve information task starts") + tabletask.LongSupplementCve() + logs.Info("END TASK:", "long Supplement cve information task is over") +} + +// ParseOpenGaussYamlTask Download the yaml file first, then parse the yaml file of opengauss +func ParseOpenGaussYamlTask() { + logs.Info("START TASK:", "the task of Process openGauss yaml data start") + tabletask.ProcGaussYaml() + logs.Info("END TASK:", "the task of Process openGauss yaml data complete") +} + +// ParseMindsporeYamlTask Download the yaml file first, then parse the yaml file of mindspore +func ParseMindsporeYamlTask() { + logs.Info("START TASK:", "the task of Process mindspore yaml data start") + tabletask.ProcMindSporeYaml() + logs.Info("END TASK:", "the task of Process mindspore yaml data complete") +} diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go new file mode 100644 index 0000000000000000000000000000000000000000..b770868aa30bc5b2d25cec4e13f3b3287143952c --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -0,0 +1,106 @@ +package controller + +import ( + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/robfig/cron/v3" +) + +func StartTimerTask() error { + c := cron.New() + + //Warehouse, organization, and owner correspondence timing task + IdGwController, err := c.AddFunc(beego.AppConfig.String("timedtask::GwController"), GwController) + if err != nil { + logs.Error("add GwController task error:", err.Error()) + return err + } + logs.Info("Add GwController task success, ID is:", IdGwController) + + IdRunTableTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunTableTask"), RunTableTask) + if err != nil { + logs.Error("add RunTableTask task error:", err.Error()) + return err + } + logs.Info("Add RunTableTask task success, ID is:", IdRunTableTask) + + //Error data export and email sending timing task + IdToExcelController, err := c.AddFunc(beego.AppConfig.String("timedtask::ToExcelController"), ToExcelController) + if err != nil { + logs.Error("add ToExcelController task error:", err.Error()) + return err + } + logs.Info("Add ToExcelController task success, ID is:", IdToExcelController) + + //Package name correspondence relationship timing task + IdRunMappingTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunMappingTask"), RunMappingTask) + if err != nil { + logs.Error("add RunMappingTask task error:", err.Error()) + return err + } + logs.Info("Add RunMappingTask task success, ID is:", IdRunMappingTask) + + //Delete ,iles timed tasks that are more than one month old + IdRunDelFileTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunDelFileTask"), RunDelFileTask) + if err != nil { + logs.Error("add RunDelFileTask task error:", err.Error()) + return err + } + logs.Info("Add RunDelFileTask task success, ID is:", IdRunDelFileTask) + + //issue record email feedback + IdRunIssueRecordTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunIssueRecordTask"), RunIssueRecordTask) + if err != nil { + logs.Error("add RunIssueRecordTask task error:", err.Error()) + return err + } + logs.Info("Add RunIssueRecordTask task success, ID is:", IdRunIssueRecordTask) + + //analysis the table package whitelist + IdRunWhitelistTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunWhitelistTask"), RunWhitelistTask) + if err != nil { + logs.Error("add RunWhitelistTask task error:", err.Error()) + return err + } + logs.Info("Add RunWhitelistTask task success, ID is:", IdRunWhitelistTask) + + //parsing the table package_committer_excels + IdRunPackageCommitterTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunPackageCommitterTask"), RunPackageCommitterTask) + if err != nil { + logs.Error("add RunPackageCommitterTask task error:", err.Error()) + return err + } + logs.Info("Add RunPackageCommitterTask task success, ID is:", IdRunPackageCommitterTask) + + //Complete the template information of the issue with the data on the CVE official website + IdSupplementCveTask, err := c.AddFunc(beego.AppConfig.String("timedtask::SupplementCveTask"), SupplementCveTask) + if err != nil { + logs.Error("add SupplementCveTask task error:", err.Error()) + return err + } + logs.Info("Add SupplementCveTask task success, ID is:", IdSupplementCveTask) + + IdLongSupplementCveTask, err := c.AddFunc(beego.AppConfig.String("timedtask::LongSupplementCveTask"), LongSupplementCveTask) + if err != nil { + logs.Error("add LongSupplementCveTask task error:", err.Error()) + return err + } + logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) + //Parse opengauss yaml file + IdParseOpenGaussYamlTask, err := c.AddFunc(beego.AppConfig.String("timedtask::ParseOpenGaussYamlTask"), ParseOpenGaussYamlTask) + if err != nil { + logs.Error("add ParseOpenGaussYamlTask task error:", err.Error()) + return err + } + logs.Info("Add ParseOpenGaussYamlTask task success, ID is:", IdParseOpenGaussYamlTask) + + //Parse the yaml file of mindspore + IdParseMindsporeYamlTask, err := c.AddFunc(beego.AppConfig.String("timedtask::ParseMindsporeYamlTask"), ParseMindsporeYamlTask) + if err != nil { + logs.Error("add ParseMindsporeYamlTask task error:", err.Error()) + return err + } + logs.Info("Add ParseMindsporeYamlTask task success, ID is:", IdParseMindsporeYamlTask) + c.Start() + return nil +} diff --git a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go new file mode 100644 index 0000000000000000000000000000000000000000..065dd56aca08961be64ebd35497fe0601e14aa4c --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go @@ -0,0 +1,457 @@ +package db_models + +import ( + "cvevulner/models" + "github.com/astaxie/beego/orm" + "time" +) + +// UpdateYamlOriginMark Update origin data +func UpdateYamlOriginMark() error { + sqlString := "update cve_open_guss_yaml set mark_bit = 2 where mark_bit = 1" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +// SelectYamlOriginData Query origin data +func SelectYamlOriginData(packageName, version string) (models.OpenGussYaml, error) { + var openGussYaml models.OpenGussYaml + sqlString := "SELECT id FROM cve_open_guss_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&openGussYaml) + return openGussYaml, err +} + +// InsertYamlOriginData insert origin data +func InsertYamlOriginData(openGussYaml *models.OpenGussYaml) error { + sqlString := "insert into cve_open_guss_yaml(package_name,version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) values(?,?,?,?,?,?,?,?,?)" + _, err := ORM.Raw(sqlString, openGussYaml.PackageName, openGussYaml.Version, openGussYaml.OriginUrl, openGussYaml.Status, openGussYaml.CpeName, openGussYaml.CreateTime, openGussYaml.UpdateTime, openGussYaml.MarkBit, openGussYaml.Repo).Exec() + return err +} + +// UpdateYamlOriginData Update origin data +func UpdateYamlOriginData(openGussYaml *models.OpenGussYaml) (int64, error) { + id, err := ORM.Update(openGussYaml, "OriginUrl", "Status", "CpeName", "UpdateTime", "Repo", "MarkBit") + return id, err +} + +// SelectYamlData Query data +func SelectYamlData(packageName, version string) ([]models.GitOpenEuler, int64, error) { + var gitOpenEulerList []models.GitOpenEuler + sqlString := "SELECT package_id, git_id FROM cve_git_open_euler WHERE package_name = ? and version = ? order by package_id desc" + rowsAffected, err := ORM.Raw(sqlString, packageName, version).QueryRows(&gitOpenEulerList) + return gitOpenEulerList, rowsAffected, err +} + +// SelectYamlLastData Query the last data +func SelectYamlLastData() (models.GitOpenEuler, error) { + var gitOpenEuler models.GitOpenEuler + sqlString := "SELECT package_id FROM cve_git_open_euler order by git_id desc" + err := ORM.Raw(sqlString).QueryRow(&gitOpenEuler) + return gitOpenEuler, err +} + +// InsertYamlData insert data +func InsertYamlData(gitOpenEuler *models.GitOpenEuler) (int64, error) { + sqlString := "insert into cve_git_open_euler(package_id,package_name,version,origin_url,create_time,update_time,cpe_packname,status) values(?,?,?,?,?,?,?,?)" + result, err := ORM.Raw(sqlString, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version, gitOpenEuler.OriginUrl, gitOpenEuler.CreateTime, gitOpenEuler.UpdateTime, gitOpenEuler.CpePackName, gitOpenEuler.Status).Exec() + id, _ := result.LastInsertId() + return id, err +} + +// InsertYamlDetailData insert data details +func InsertYamlDetailData(gitPackageInfo *models.GitPackageInfo) error { + sqlString := "insert into cve_git_package_info(git_id,package_name,version,origin_url,create_time,update_time,decription,status) values(?,?,?,?,?,?,?,?)" + _, err := ORM.Raw(sqlString, gitPackageInfo.GitId, gitPackageInfo.PackageName, gitPackageInfo.Version, gitPackageInfo.OriginUrl, gitPackageInfo.CreateTime, gitPackageInfo.UpdateTime, gitPackageInfo.Decription, gitPackageInfo.Status).Exec() + return err +} + +// DeleteYamlOpenEulerDetailData delete origin data +func DeleteYamlOpenEulerDetailData(gitId int64) error { + sqlString := "delete from cve_git_package_info where git_id = ?" + _, err := ORM.Raw(sqlString, gitId).Exec() + return err +} + +// DeleteYamlOpenEulerData delete origin data +func DeleteYamlOpenEulerData(gitId int64) error { + sqlString := "delete from cve_git_open_euler where git_id = ?" + _, err := ORM.Raw(sqlString, gitId).Exec() + return err +} + +// UpdateYamlData update data +func UpdateYamlData(gitOpenEuler *models.GitOpenEuler) error { + sqlString := "update cve_git_open_euler set origin_url = ?,update_time = ?,cpe_packname = ?,status=? where package_id = ? and package_name = ? and version = ?" + _, err := ORM.Raw(sqlString, gitOpenEuler.OriginUrl, gitOpenEuler.UpdateTime, gitOpenEuler.CpePackName, gitOpenEuler.Status, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version).Exec() + return err +} + +// UpdateYamlDetailData Update data details +func UpdateYamlDetailData(gitPackageInfo *models.GitPackageInfo) error { + sqlString := "update cve_git_package_info set package_name = ?,version = ?,origin_url = ?, update_time = ?,status=? where git_id = ?" + _, err := ORM.Raw(sqlString, gitPackageInfo.PackageName, gitPackageInfo.Version, gitPackageInfo.OriginUrl, gitPackageInfo.UpdateTime, gitPackageInfo.Status, gitPackageInfo.GitId).Exec() + return err +} + +// DeleteYamlOriginMark delete origin data +func DeleteYamlOriginMark() error { + sqlString := "delete from cve_open_guss_yaml where mark_bit = 2" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +// SelectOpenEulerYamlData Query data +func SelectOpenEulerYamlData() ([]models.GitOpenEuler, int64, error) { + var gitOpenEulerList []models.GitOpenEuler + sqlString := "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler WHERE package_id >= ?" + rowsAffected, err := ORM.Raw(sqlString, 10000000).QueryRows(&gitOpenEulerList) + return gitOpenEulerList, rowsAffected, err +} + +// SelectGaussYamlOriginData Query origin data +func SelectGaussYamlOriginData(packageName, version string) (models.OpenGussYaml, error) { + var openGussYaml models.OpenGussYaml + sqlString := "SELECT id FROM cve_open_guss_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&openGussYaml) + return openGussYaml, err +} + +// SelectMindSporeYamlOriginData Query origin data +func SelectMindSporeYamlOriginData(packageName, version string) (models.MindSporeYaml, error) { + var mindSporeYaml models.MindSporeYaml + sqlString := "SELECT id FROM cve_mind_spore_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&mindSporeYaml) + return mindSporeYaml, err +} + +// SelectOpenLookengYamlOriginData Query origin data +func SelectOpenLookengYamlOriginData(packageName, version string) (models.OpenLookengYaml, error) { + var openLookengYaml models.OpenLookengYaml + sqlString := "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&openLookengYaml) + return openLookengYaml, err +} + +func GetOneGitRepoGroups(sigName string) (models.GitRepoGroups, error) { + var gitRepsGroups models.GitRepoGroups + sqlString := "select * from cve_git_repo_groups where group_name = ?" + err := ORM.Raw(sqlString, sigName).QueryRow(&gitRepsGroups) + return gitRepsGroups, err +} + +func InsertOneGitRepoGroups(gitRepoGroups *models.GitRepoGroups) error { + sqlString := "insert into cve_git_repo_groups (group_name) values (?)" + _, err := ORM.Raw(sqlString, gitRepoGroups.GroupName).Exec() + return err +} + +func DeleteGitRepoMember(groupId int64) error { + sqlString := "delete from cve_gite_repo_member where group_id = ?" + _, err := ORM.Raw(sqlString, groupId).Exec() + return err +} + +func InsertOneGiteRepoMember(giteRepoMember *models.GiteRepoMember) error { + sqlString := "insert into cve_gite_repo_member (group_id,member_name,member_type,create_time) values (?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, giteRepoMember.GroupId, giteRepoMember.MemberName, giteRepoMember.MemberType, giteRepoMember.CreateTime).Exec() + return err +} + +func GetCveScoreListByBeforeDate(beforeDate string) ([]models.Score, int64, error) { + var scoreList []models.Score + sqlString := "select cve_id,cve_num from cve_score where nvd_score = 0 and update_time >= ? order by update_time desc" + rowsAffected, err := ORM.Raw(sqlString, beforeDate).QueryRows(&scoreList) + return scoreList, rowsAffected, err +} + +func GetCveVulnCenterList(cveDesc, repairTime, updateTime string) ([]models.VulnCenter, int64, error) { + var vulnCenterList []models.VulnCenter + sqlString := "select cve_id,cve_num from cve_vuln_center where (cve_desc = ? or repair_time = ?) and update_time >= ? order by cve_id desc" + rowsAffected, err := ORM.Raw(sqlString, cveDesc, repairTime, updateTime).QueryRows(&vulnCenterList) + return vulnCenterList, rowsAffected, err +} + +func GetCveIssueTemplate(nvdScore, nvdVector, cveBrief, updateTime string) ([]models.IssueTemplate, int64, error) { + var issueTemplateList []models.IssueTemplate + sqlString := "select cve_id,cve_num from cve_issue_template where (nvd_score = ? or nvd_vector = ? or cve_brief = ?) and update_time >= ? order by cve_id desc" + rowsAffected, err := ORM.Raw(sqlString, nvdScore, nvdVector, cveBrief, updateTime).QueryRows(&issueTemplateList) + return issueTemplateList, rowsAffected, err +} + +func GetOneVulnCenter(cveId, cveNum string) (models.VulnCenter, error) { + var VulnCenter models.VulnCenter + sqlString := "select cve_num, pack_name, cve_version, cve_desc,repair_time,cve_status,cve_id from cve_vuln_center where cve_id = ? and cve_num = ? order by cve_id desc" + err := ORM.Raw(sqlString, cveId, cveNum).QueryRow(&VulnCenter) + return VulnCenter, err +} + +func UpdateVulnCenterTypeOne(vulnCenter models.VulnCenter) error { + sqlString := "update cve_vuln_center set repair_time = ?,update_time=?, cve_status=? where cve_id=?" + _, err := ORM.Raw(sqlString, vulnCenter.RepairTime, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() + return err +} + +func UpdateVulnCenterTypeTwo(vulnCenter models.VulnCenter) error { + sqlString := "update cve_vuln_center set cve_desc = ?,update_time=?, cve_status=? where cve_id=?" + _, err := ORM.Raw(sqlString, vulnCenter.Description, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() + return err +} + +func GetCveScoreByCveIdAndCveNum(cveId int64, cveNum string) (models.Score, error) { + var score models.Score + sqlString := "select id,nvd_score from cve_score where cve_id = ? and cve_num = ?" + err := ORM.Raw(sqlString, cveId, cveNum).QueryRow(&score) + return score, err +} + +func UpdateCveScore(score models.Score) error { + sqlString := "update cve_score set nvd_score=?, n_vector_value=?, n_attack_vector=?, n_access_vector=?, n_attack_complexity=?, n_access_complexity=?, n_privilege_required=?, n_user_interaction=?, n_scope=?, n_confidentiality=?, n_integrity=?, n_availability=?, n_authentication=?, update_time=?,score_type=? where id = ?" + _, err := ORM.Raw(sqlString, score.NVDScore, score.NvectorVule, score.NattackVector, score.NaccessVector, score.NattackComplexity, score.NaccessComplexity, score.NprivilegeRequired, score.NuserInteraction, score.Nscope, score.Nconfidentiality, score.Nintegrity, score.Navailability, score.Nauthentication, score.UpdateTime, score.ScoreType, score.Id).Exec() + return err +} + +func GetIssueTemplateTypeOne(issueTemplate models.IssueTemplate) (models.IssueTemplate, error) { + var issueTemplateResp models.IssueTemplate + sqlString := "select template_id,nvd_score,nvd_vector,cve_brief from cve_issue_template where (nvd_score = ? or nvd_vector = ? or cve_brief = ?) and cve_id = ? and cve_num = ?" + err := ORM.Raw(sqlString, issueTemplate.NVDScore, issueTemplate.NVDVector, issueTemplate.CveBrief, issueTemplate.CveId, issueTemplate.CveNum).QueryRow(&issueTemplateResp) + return issueTemplateResp, err +} + +func UpdateCveIssueTemplateTypeOne(issueTemplate models.IssueTemplate) error { + sqlString := "update cve_issue_template set nvd_score = ?, update_time=? where template_id = ?" + _, err := ORM.Raw(sqlString, issueTemplate.NVDScore, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() + return err +} + +func UpdateCveIssueTemplateTypeTwo(issueTemplate models.IssueTemplate) error { + sqlString := "update cve_issue_template set nvd_vector = ?, update_time=? where template_id = ? " + _, err := ORM.Raw(sqlString, issueTemplate.NVDVector, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() + return err +} + +func UpdateCveIssueTemplateTypeThree(issueTemplate models.IssueTemplate) error { + sqlString := "update cve_issue_template set cve_brief = ?, update_time=? where template_id = ? " + _, err := ORM.Raw(sqlString, issueTemplate.CveBrief, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() + return err +} + +func UpdateVulnCenter(vulnCenter models.VulnCenter) error { + sqlString := "update cve_vuln_center set update_time=?, cve_status=? where cve_id=?" + _, err := ORM.Raw(sqlString, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() + return err +} + +func GetSpceError(cveNum, cveOwner, packName string) (models.SpecError, error) { + var spceError models.SpecError + sqlString := "select * from cve_spec_error where cve_num = ? and cve_owner = ? and pack_name = ?" + err := ORM.Raw(sqlString, cveNum, cveOwner, packName).QueryRow(&spceError) + return spceError, err +} + +func UpdateCveOriginExcel(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set cve_desc = ?, cve_status = ? where cve_num= ? and pack_name = ? and cve_version = ?" + _, err := ORM.Raw(sqlString, originExcel.CveDesc, originExcel.CveStatus, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() + return err +} + +// UpdateMindYamlOriginMark UpdateYamlOriginMarkLookeng Update origin data +func UpdateMindYamlOriginMark() error { + sqlString := "update cve_open_lookeng_yaml set mark_bit = 2 where mark_bit = 1" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +// SelectMindYamlOriginData Query origin data +func SelectMindYamlOriginData(packageName, version, repokey string) (models.OpenGussYaml, error) { + var openGussYaml models.OpenGussYaml + sqlString := "SELECT id FROM cve_mind_spore_yaml WHERE package_name = ? and version = ? and repo_name = ?" + err := ORM.Raw(sqlString, packageName, version, repokey).QueryRow(&openGussYaml) + return openGussYaml, err +} + +// InsertMindYamlOriginData insert origin data +func InsertMindYamlOriginData(mindSporeYaml *models.MindSporeYaml) error { + sqlString := "insert into cve_mind_spore_yaml(package_name,version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) values(?,?,?,?,?,?,?,?,?)" + _, err := ORM.Raw(sqlString, mindSporeYaml.PackageName, mindSporeYaml.Version, mindSporeYaml.OriginUrl, mindSporeYaml.Status, mindSporeYaml.CpeName, mindSporeYaml.CreateTime, mindSporeYaml.UpdateTime, mindSporeYaml.MarkBit, mindSporeYaml.Repo).Exec() + return err +} + +// UpdateMindYamlOriginData Update origin data +func UpdateMindYamlOriginData(mindSporeYaml *models.MindSporeYaml) (int64, error) { + id, err := ORM.Update(mindSporeYaml, "OriginUrl", "Status", "CpeName", "UpdateTime", "MarkBit") + return id, err +} + +// DeleteMindYamlOriginMark delete origin data +func DeleteMindYamlOriginMark() error { + sqlString := "delete from cve_mind_spore_yaml where mark_bit = 2" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +func SelectCveFileHashByFileName(fileName string) ([]interface{}, int64, error) { + var list orm.ParamsList + sqlString := "select file_hash from cve_file_hash where file_name = ?" + num, err := ORM.Raw(sqlString, fileName).ValuesFlat(&list) + return list, num, err +} + +func InsertCveFileHash(fileName, fileHash string) error { + sqlString := "insert into cve_file_hash (file_name, file_hash) values (?, ?)" + _, err := ORM.Raw(sqlString, fileName, fileHash).Exec() + return err +} + +func InsertSpecIssueAssigness(specIssueAssigness *models.SpecIssueAssigness) error { + sqlString := "insert ignore into cve_spec_issue_assigness (package_name, issue_assignee, status, create_time) values (?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, specIssueAssigness.PackageName, specIssueAssigness.Assignee, specIssueAssigness.Status, specIssueAssigness.CreateTime).Exec() + return err +} + +func SelectIssueRepoWhiteListByPackageName(packageName string) (models.IssueRepoWhitelist, error) { + var issueRepoWhiteList models.IssueRepoWhitelist + sqlString := "select * from cve_issue_repo_whitelist where package_name = ?" + err := ORM.Raw(sqlString, packageName).QueryRow(&issueRepoWhiteList) + return issueRepoWhiteList, err +} + +func InsertIssueRepoWhiteList(issueRepoWhiteList *models.IssueRepoWhitelist) error { + sqlString := "insert into cve_issue_repo_whitelist (package_name, version, status, branchs, create_time, update_time, delete_time) values (?, ?, ?, ?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, issueRepoWhiteList.PackageName, issueRepoWhiteList.Version, issueRepoWhiteList.Status, issueRepoWhiteList.Branchs, issueRepoWhiteList.CreateTime, issueRepoWhiteList.UpdateTime, issueRepoWhiteList.DeleteTime).Exec() + return err +} + +func UpdateIssueRepoWhiteList(issueRepoWhiteList models.IssueRepoWhitelist) error { + sqlString := "update cve_issue_repo_whitelist set status = ?, branchs = ?, update_time = ? where package_name = ? and version = ?" + _, err := ORM.Raw(sqlString, issueRepoWhiteList.Status, issueRepoWhiteList.Branchs, issueRepoWhiteList.UpdateTime, issueRepoWhiteList.PackageName, issueRepoWhiteList.Version).Exec() + return err +} + +func GetResults(status string) ([]models.IssueCreateRecord, int64, error) { + lastMonthDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") + var issueCreateRecordList []models.IssueCreateRecord + sqlString := "select * from cve_issue_create_record where status = ? and create_time > ?" + rowsAffected, err := ORM.Raw(sqlString, status, lastMonthDate).QueryRows(&issueCreateRecordList) + return issueCreateRecordList, rowsAffected, err +} + +func UpdateStatue(status string) error { + sqlString := "update cve_issue_create_record set status = ?,update_time = ? where status = ?" + _, err := ORM.Raw(sqlString, 4, time.Now().Format("2006-01-02 15:04:05"), status).Exec() + return err +} + +func GetCveEmailListByEmailType(emailType string) ([]models.EmailList, int64, error) { + var emailList []models.EmailList + sqlString := "select email_name from cve_email_list where email_type=?" + rowsAffected, err := ORM.Raw(sqlString, emailType).QueryRows(&emailList) + return emailList, rowsAffected, err +} + +func SelectCvePackAgeCpeByPackName(packName string) (models.PackageCpe, error) { + var packageCpe models.PackageCpe + sqlString := "select * from cve_package_cpe where packname = ?" + err := ORM.Raw(sqlString, packName).QueryRow(&packageCpe) + return packageCpe, err +} + +func InsertCvePackAgeCpe(packageCpe *models.PackageCpe) error { + sqlString := "insert into cve_package_cpe (packname,cpe_packname,create_time) values (?,?,?)" + _, err := ORM.Raw(sqlString, packageCpe.PackName, packageCpe.CpePackName, packageCpe.CreateTime).Exec() + return err +} + +func UpdateCvePackAgeCpe(packageCpe models.PackageCpe) error { + sqlString := "update cve_package_cpe set cpe_packname=?, create_time=? where packname=?" + _, err := ORM.Raw(sqlString, packageCpe.CpePackName, packageCpe.CreateTime, packageCpe.PackName).Exec() + return err +} + +func GetCveOriginExcel() ([]models.OriginExcel, int64, error) { + lastMonthDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") + var originExcel []models.OriginExcel + sqlString := "select * from cve_origin_excel where (cve_status = 3 or cve_status=4) and is_export = 1 and create_time > ?" + rowsAffected, err := ORM.Raw(sqlString, lastMonthDate).QueryRows(&originExcel) + return originExcel, rowsAffected, err +} + +func UpdateStatusTypeTwo() error { + sqlString := "update cve_origin_excel set is_export=2 where cve_status=3 or cve_status=4" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +func GetCveSpecError(cveNum string) (models.SpecError, error) { + var specError models.SpecError + sqlString := "SELECT * FROM cve_spec_error where cve_num = ?" + err := ORM.Raw(sqlString, cveNum).QueryRow(&specError) + return specError, err +} + +func InsertCveSpecError(specError *models.SpecError) (int64, error) { + return ORM.Insert(specError) +} + +func UpdateCveSpecError(specError models.SpecError) error { + sqlString := "update cve_spec_error set cve_desc = ?, cve_owner = ?, cve_status = ?, pack_name = ?, update_time = ? where cve_num = ?" + _, err := ORM.Raw(sqlString, specError.Description, specError.Owner, specError.Status, specError.PackName, specError.UpdateTime, specError.CveNum).Exec() + return err +} + +func GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion string) (models.OriginExcel, error) { + var originExcel models.OriginExcel + sqlString := "select * from cve_origin_excel where cve_num= ? and pack_name = ? and cve_version = ?" + err := ORM.Raw(sqlString, cveNum, packName, cveVersion).QueryRow(&originExcel) + return originExcel, err +} + +func UpdateCveOriginExcelTypeThree(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set nvd_score=?, cve_level=?, cve_desc=?, repair_time=?, vector_value=?, attack_vector=?, access_vector=?, attack_complexity=?, access_complexity=?, privilege_required=?, user_interaction=?, scope=?, confidentiality=?, integrity=?, availability=?, authentication=?, cve_status=?, update_time=? where cve_num=? and pack_name = ? and cve_version = ?" + _, err := ORM.Raw(sqlString, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.UpdateTime, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() + return err +} + +func UpdateCveOriginExcelTypeFour(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set nvd_score=?, cve_level=?, cve_desc=?, repair_time=?, vector_value=?, attack_vector=?, access_vector=?, attack_complexity=?, access_complexity=?, privilege_required=?, user_interaction=?, scope=?, confidentiality=?, integrity=?, availability=?, authentication=?, cve_status=?, update_time=?, score_type=? where cve_num=? and pack_name = ? and cve_version = ?" + _, err := ORM.Raw(sqlString, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.UpdateTime, originExcel.ScoreType, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() + return err +} + +func UpdateCveOriginExcelTypeFive(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set cve_desc = ?, cve_status = ? where cve_num = ?" + _, err := ORM.Raw(sqlString, originExcel.CveDesc, originExcel.CveStatus, originExcel.CveNum).Exec() + return err +} + +func InsertCveOriginExcel(originExcel *models.OriginExcel) error { + sqlString := "INSERT INTO cve_origin_excel (cve_num, cve_url, cve_version, pack_name, score_type, nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, access_vector, attack_complexity, access_complexity, privilege_required, user_interaction, scope, confidentiality, integrity, availability, authentication, cve_status, create_time, update_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, originExcel.CveNum, originExcel.CveUrl, originExcel.CveVersion, originExcel.PackName, originExcel.ScoreType, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.CreateTime, originExcel.UpdateTime).Exec() + return err +} + +func SelectCveOriginExcel(scoreType string) ([]models.OriginExcel, int64, error) { + var originExcel []models.OriginExcel + sqlString := "SELECT cve_num,pack_name,cve_version FROM cve_origin_excel WHERE score_type = ? OR score_type IS NULL OR cve_desc IS NULL OR nvd_score IS NULL" + rowsAffected, err := ORM.Raw(sqlString, scoreType).QueryRows(&originExcel) + return originExcel, rowsAffected, err +} + +func SelectCveVulnCenter() ([]models.VulnCenter, int64, error) { + var vulnCenter []models.VulnCenter + sqlString := "SELECT cve_id, cve_num, repair_time FROM cve_vuln_center WHERE repair_time = '' OR repair_time IS NULL" + rowsAffected, err := ORM.Raw(sqlString).QueryRows(&vulnCenter) + return vulnCenter, rowsAffected, err +} + +func UpdateCveVulnCenter(repairTime, cveId string) error { + sqlString := "update cve_vuln_center set repair_time = ? where cve_id = ?" + _, err := ORM.Raw(sqlString, repairTime, cveId).Exec() + return err +} + +func QueryCveByNumber(cveNum string) ([]models.VulnCenter, int64, error) { + var vulnCenter []models.VulnCenter + sqlString := "select cve_num, pack_name, cve_version, cve_desc,repair_time,cve_status,cve_id from cve_vuln_center where cve_num = ? order by cve_id desc" + rowsAffected, err := ORM.Raw(sqlString, cveNum).QueryRows(&vulnCenter) + return vulnCenter, rowsAffected, err +} diff --git a/cve-vulner-manager/cve-timed-task/db_models/init_db.go b/cve-vulner-manager/cve-timed-task/db_models/init_db.go new file mode 100644 index 0000000000000000000000000000000000000000..7692a99bcfc32b798f7d22c73285aab8af157cd1 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/db_models/init_db.go @@ -0,0 +1,13 @@ +package db_models + +import ( + "github.com/astaxie/beego/orm" + _ "github.com/go-sql-driver/mysql" +) + +var ORM orm.Ormer + +func InitTimedTaskDB() { + myOrm := orm.NewOrm() + ORM = myOrm +} diff --git a/cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go b/cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go new file mode 100644 index 0000000000000000000000000000000000000000..76429198d9ee6bfcd6d34d10b4fdd02ba7fa8e2b --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go @@ -0,0 +1,48 @@ +package deletetask + +import ( + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" + "time" +) + +func DelDir(filePath string) { + + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + return + } + + for _, file := range fileList { + fileName := file.Name() + if file.IsDir() { + childFilePath := filePath + fileName + "/" + childFileList, err := ioutil.ReadDir(childFilePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + } + if len(childFileList) == 0 { + logs.Info("this is an empty folder", childFilePath) + err = os.Remove(childFilePath) + if err != nil { + logs.Error("remove empty folder filePath error:", childFilePath, err.Error()) + } + return + } + DelDir(childFilePath) + } + now := time.Now() + if now.Unix()-60*60*24*30 > file.ModTime().Unix() { + removePath := filePath + fileName + err = os.Remove(removePath) + if err != nil { + logs.Error("remove file removePath error:", err.Error()) + } + logs.Info(removePath, "is remove at", now) + } + + } + +} diff --git a/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go b/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go new file mode 100644 index 0000000000000000000000000000000000000000..b58af19153b4157a2db4b3be5afa22aaced6ffd8 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go @@ -0,0 +1,315 @@ +package downloadfiletask + +import ( + "cvevulner/cve-timed-task/util" + "fmt" + "github.com/antchfx/htmlquery" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io" + "net/http" + "os" + "strings" +) + +func HandleOne() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "newexcels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/newexcels" + html, err := util.UrlToHTML(url) + if err != nil { + logs.Error("util.UrlToHTML error: ", err, url) + return "", err + } + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("htmlquery.QueryAll error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/newexcels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil + +} + +func HandleTwo() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "mappingexcels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/mappingexcels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/mappingexcels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} + +func HandleThree() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "import_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/import_excels" + + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/import_excels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + _ = resp.Body.Close() + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} + +func HandleFour() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "package_committer_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/package_committer_excels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/package_committer_excels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + _ = resp.Body.Close() + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} + +// DownloadSpecErrorExcels download spec_error_excels +func DownloadSpecErrorExcels() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "spec_error_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/spec_error_excels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("htmlquery.QueryAll(:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/spec_error_excels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go b/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go new file mode 100644 index 0000000000000000000000000000000000000000..beebeed56a401ce24f8738ae2ce3108c7b249f44 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go @@ -0,0 +1,42 @@ +package emailtask + +import ( + "cvevulner/cve-timed-task/db_models" + "github.com/astaxie/beego/logs" + "io/ioutil" +) + +// Sending Query all recipients +func Sending(filePath string) { + result, _, err := db_models.GetCveEmailListByEmailType("1") + if err != nil { + logs.Error("db_models.GetCveEmailListByEmailType error:", err) + return + } + + fList := make([]string, 0, 10) + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + } + + for _, file := range fileList { + if file.IsDir() { + continue + } + fileName := file.Name() + childFilePath := filePath + fileName + fList = append(fList, childFilePath) + } + + subject := "This time cve creates an issue record" + content := "The email attachment may contain two attached excel documents, which are the issue data that has been successfully created and the issue data that has not been created." + + for _, i := range result { + err = SendEmail([]string{i.EmailName}, []string{}, subject, content, fList) + if err != nil { + logs.Error("SendEmail error:", err) + } + } + +} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/send_email.go b/cve-vulner-manager/cve-timed-task/emailtask/send_email.go new file mode 100644 index 0000000000000000000000000000000000000000..fa41765fee4a1dbb91a8f90b41a0539de8302d43 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/emailtask/send_email.go @@ -0,0 +1,41 @@ +package emailtask + +import ( + "github.com/astaxie/beego" + "gopkg.in/gomail.v2" + "strconv" +) + +func SendEmail(mailTo, mailCc []string, subject string, body string, filePath []string) error { + emailName := beego.AppConfig.String("email::email_name") + emailPwd := beego.AppConfig.String("email::email_pwd") + emailHost := beego.AppConfig.String("email::email_host") + emailPort := beego.AppConfig.String("email::email_port") + emailPwd = "" + + mailConn := map[string]string{ + "user": emailName, + "pass": emailPwd, + "host": emailHost, + "port": emailPort, + } + + port, _ := strconv.Atoi(mailConn["port"]) + m := gomail.NewMessage() + m.SetHeader("From", "cve-manager"+"<"+mailConn["user"]+">") + m.SetHeader("To", mailTo...) + if len(mailCc) > 0 { + m.SetHeader("Cc", mailCc...) + } + m.SetHeader("Subject", subject) + m.SetBody("text/plain", body) + + for _, v := range filePath { + m.Attach(v) + } + d := gomail.NewDialer(mailConn["host"], port, mailConn["user"], mailConn["pass"]) + err := d.DialAndSend(m) + + return err + +} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go b/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go new file mode 100644 index 0000000000000000000000000000000000000000..576852bd10e094dc11da2b6cd09bbd079fb0c7e7 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go @@ -0,0 +1,40 @@ +package emailtask + +import ( + "cvevulner/cve-timed-task/db_models" + "github.com/astaxie/beego/logs" + "io/ioutil" +) + +func SendingTypeTwo(filePath string) { + result, _, err := db_models.GetCveEmailListByEmailType("1") + if err != nil { + logs.Error("db_models.GetCveEmailListByEmailType error:", err) + return + } + + fList := make([]string, 0, 10) + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + } + + for _, file := range fileList { + if file.IsDir() { + continue + } + fileName := file.Name() + childFilePath := filePath + fileName + fList = append(fList, childFilePath) + } + + subject := "Form error data feedback(人工CVE漏洞数据错误反馈)" + content := "The submitted CVE data is wrong, please check the format and content, please refer to the attachment" + + for _, i := range result { + err = SendEmail([]string{i.EmailName}, []string{}, subject, content, fList) + if err != nil { + logs.Error("SendEmail error:", err) + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go new file mode 100644 index 0000000000000000000000000000000000000000..e817f41e01015b33e19694b71d4939f3b6b1744f --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go @@ -0,0 +1,138 @@ +package gitwebtask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "fmt" + "github.com/antchfx/htmlquery" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "strings" + "time" +) + +func Handle() { + url := "https://gitee.com/openeuler/community/tree/master/sig" + doc, err := util.UrlToHTML(url) + if err != nil { + logs.Error(" util.UrlToHTML error:", err, url) + return + } + + type sig struct { + sigName string + sigPage string + etherpad string + } + + sigsList := make([]sig, 0, 100) + for i := 3; ; i += 2 { + node, err := htmlquery.QueryAll(doc, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("htmlquery.QueryAll error:", err) + continue + } + if len(node) == 0 { + break + } + + sigName := htmlquery.SelectAttr(node[0], "title") + if sigName == "sigs.yaml" || sigName == "README.md" { + break + } + + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + _, err = db_models.GetOneGitRepoGroups(sigName) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + err = db_models.InsertOneGitRepoGroups(&models.GitRepoGroups{ + GroupName: sigName, + }) + if err != nil { + logs.Error("db_models.InsertOneGitRepoGroups error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + } + } else { + logs.Error("db_models.InsertOneGitRepoGroups error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + } + } else { + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + } + } + + sigPageNodes, err := htmlquery.QueryAll(doc, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@href", i)) + if err != nil { + logs.Warn("htmlquery.QueryAll error:", err) + continue + } + sigPage := htmlquery.SelectAttr(sigPageNodes[0], "href") + + etherpad := fmt.Sprintf("https://etherpad.openeuler.org/p/%s-meetings", sigName) + s := sig{ + sigName: sigName, + sigPage: sigPage, + etherpad: etherpad, + } + //Get the name, homepage and etherpad of all sigs + sigsList = append(sigsList, s) + } + for _, s := range sigsList { + gitRepoGroup, err := db_models.GetOneGitRepoGroups(s.sigName) + if err != nil { + logs.Warn("db_models.GetOneGitRepoGroups error:", err.Error()) + continue + } + err = db_models.DeleteGitRepoMember(gitRepoGroup.GroupId) + if err != nil { + logs.Warn("db_models.DeleteGitRepoMember error:", err.Error()) + continue + } + + urlOWNERS := fmt.Sprintf("https://gitee.com/openeuler/community/blob/master/sig/%s/OWNERS", s.sigName) + docOWNERS, err := util.UrlToHTML(urlOWNERS) + if err != nil { + logs.Error("util.UrlToHTML error:", err.Error(), url) + continue + } + nodeOWNERS, err := htmlquery.QueryAll(docOWNERS, "//div[@class=\"line\"]/text()") + if err != nil { + logs.Error("htmlquery.QueryAll error:", err) + continue + } + if len(nodeOWNERS) > 1 { + for _, k := range nodeOWNERS[1:] { + i := htmlquery.InnerText(k) + mSplit := strings.Split(i, "-") + maintainer := strings.TrimSpace(mSplit[len(mSplit)-1]) + giteRepoMember := &models.GiteRepoMember{ + GroupId: gitRepoGroup.GroupId, + MemberName: maintainer, + MemberType: "Maintainer", + CreateTime: time.Now().Format("2006-01-02 15:04:05"), + } + err = db_models.InsertOneGiteRepoMember(giteRepoMember) + if err != nil { + logs.Error("db_models.InsertOneGiteRepoMember error:", err.Error()) + continue + } + } + } + } + +} diff --git a/cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go b/cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go new file mode 100644 index 0000000000000000000000000000000000000000..a75be2c84e2770189b8fcb539d2345864b42202f --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go @@ -0,0 +1,6 @@ +package gitwebtask + +// DownLoad download detail +func DownLoad() { + +} diff --git a/cve-vulner-manager/cve-timed-task/start.go b/cve-vulner-manager/cve-timed-task/start.go new file mode 100644 index 0000000000000000000000000000000000000000..39c24cb608c909682b116e5059ade1493a02bbff --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/start.go @@ -0,0 +1,17 @@ +package cve_timed_task + +import ( + "cvevulner/cve-timed-task/controller" + "cvevulner/cve-timed-task/db_models" +) + +func StartCveTimedTask() error { + + db_models.InitTimedTaskDB() + + err := controller.StartTimerTask() + if err != nil { + return err + } + return nil +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/common.go b/cve-vulner-manager/cve-timed-task/tabletask/common.go new file mode 100644 index 0000000000000000000000000000000000000000..c8945cdf094d01c3afaf37aaefbffeb99daeec19 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/common.go @@ -0,0 +1,66 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" +) + +// DeleteYamlData delete yaml data +func DeleteYamlData() { + packagesData, rowsAffected, err := db_models.SelectOpenEulerYamlData() + if err != nil { + logs.Error("db_models.SelectOpenEulerYamlData error:", err.Error()) + return + } + if rowsAffected > 0 { + for _, v := range packagesData { + if v.PackageId >= 10000000 { + _, errg := db_models.SelectGaussYamlOriginData(v.PackageName, v.Version) + _, errm := db_models.SelectMindSporeYamlOriginData(v.PackageName, v.Version) + _, erro := db_models.SelectOpenLookengYamlOriginData(v.PackageName, v.Version) + if errors.Is(orm.ErrNoRows, errg) && errors.Is(orm.ErrNoRows, errm) && errors.Is(orm.ErrNoRows, erro) { + logs.Info("ID of the currently deleted data: ", v.GitId) + //open transaction + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + err = db_models.DeleteYamlOpenEulerDetailData(v.GitId) + if err != nil { + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + err = db_models.DeleteYamlOpenEulerData(v.GitId) + if err != nil { + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } else { + if errg != nil && !errors.Is(orm.ErrNoRows, errg) { + logs.Error("db_models.SelectGaussYamlOriginData:", errg, v.PackageName, v.Version) + } + if errm != nil && errors.Is(orm.ErrNoRows, errm) { + logs.Error("db_models.SelectMindSporeYamlOriginData", errm, v.PackageName, v.Version) + } + if erro != nil && errors.Is(orm.ErrNoRows, erro) { + logs.Error("db_models.SelectOpenLookengYamlOriginData", erro, v.PackageName, v.Version) + } + } + } + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go new file mode 100644 index 0000000000000000000000000000000000000000..da06e58fea0e8bca3e8686b49bfcbbda0cd113f2 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go @@ -0,0 +1,301 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/util" + "github.com/antchfx/htmlquery" + "github.com/astaxie/beego/logs" + "github.com/pkg/errors" + "strings" + "time" +) + +type XpathList struct { + NvdScore string + CveLevel string + CveDesc string + RepairTime string + VectorValue string + AttackVector string + AccessVector string + AttackComplexity string + AccessComplexity string + PrivilegeRequired string + UserInteraction string + Scope string + Confidentiality string + Integrity string + Availability string + Authentication string + ScoreType string +} + +// Crawling Grab cve specific information +func Crawling(url string) (XpathList, error) { + xpathList := XpathList{} + + if url == "" || !strings.Contains(url, "http") { + logs.Error("url is not standardized:", url) + return xpathList, errors.New("url is not standardized") + } + + html, err := util.UrlToHTML(url) + if err != nil { + logs.Error("util.UrlToHTML error:", err.Error(), url) + return xpathList, err + } + //xpathList.CveDesc + nodes, err := htmlquery.QueryAll(html, "//*[@id=\"vulnDetailTableView\"]//tr/td/div/div[1]/p[1]/text()") + if err != nil { + return xpathList, err + } else { + if len(nodes) > 0 { + cveDesc := htmlquery.InnerText(nodes[0]) + if cveDesc == "N/A" { + xpathList.CveDesc = "" + } else { + xpathList.CveDesc = cveDesc + } + } + } + //xpathList.RepairTime + nodes, err = htmlquery.QueryAll(html, "//*[@data-testid=\"vuln-published-on\"]/text()") + if err != nil { + return xpathList, err + } else { + if len(nodes) > 0 { + repairTime := htmlquery.InnerText(nodes[0]) + if repairTime != "" { + if repairTime == "N/A" { + xpathList.RepairTime = "" + } else { + w, err := time.ParseInLocation("01/02/2006", repairTime, time.Local) + if err == nil { + xpathList.RepairTime = w.Format("2006-01-02 15:04:05") + } else { + logs.Error(err.Error()) + } + } + } + } + } + xpathList.ScoreType = "v3.0" + + nodes, err = htmlquery.QueryAll(html, "//*[@id=\"nistV3MetricHidden\"]/@value") + if err != nil { + return xpathList, err + } + if len(nodes) == 0 { + nodes, err = htmlquery.QueryAll(html, "//*[@id=\"cnaV3MetricHidden\"]/@value") + if err != nil { + return xpathList, err + } + } + if len(nodes) > 0 { + value := htmlquery.SelectAttr(nodes[0], "value") + if value != "" { + childHtml, err := htmlquery.Parse(strings.NewReader(value)) + if err != nil { + return xpathList, err + } + + //xpathList.CveLevel + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-base-score-severity\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + cl := htmlquery.InnerText(nodes[0]) + if len(cl) > 0 { + cl = strings.TrimSpace(cl) + xpathList.CveLevel = strings.ToUpper(cl[:1]) + strings.ToLower(cl[1:]) + } + } + + //xpathList.NvdScore + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-base-score\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.NvdScore = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.VectorValue + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-vector\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + vv := htmlquery.InnerText(nodes[0]) + vv = strings.Replace(vv, "(", "", -1) + vv = strings.Replace(vv, ")", "", -1) + xpathList.VectorValue = strings.TrimSpace(vv) + } + //xpathList.AttackVector + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-av\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AttackVector = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.AttackComplexity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-ac\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AttackComplexity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.PrivilegeRequired + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-pr\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.PrivilegeRequired = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.UserInteraction + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-ui\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.UserInteraction = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Scope + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-s\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Scope = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Confidentiality + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-c\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Confidentiality = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Integrity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-i\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Integrity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Integrity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-a\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Availability = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + } + } else { + nodes, err = htmlquery.QueryAll(html, "//*[@id=\"nistV2MetricHidden\"]/@value") + if len(nodes) > 0 { + value := htmlquery.SelectAttr(nodes[0], "value") + if value != "" { + childHtml, err := htmlquery.Parse(strings.NewReader(value)) + if err != nil { + return xpathList, err + } + xpathList.ScoreType = "v2.0" + //xpathList.CveLevel + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-base-score-severity\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + cl := htmlquery.InnerText(nodes[0]) + if len(cl) > 0 { + cl = strings.TrimSpace(cl) + xpathList.CveLevel = strings.ToUpper(cl[:1]) + strings.ToLower(cl[1:]) + } + } + + //xpathList.NvdScore + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-base-score\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.NvdScore = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.VectorValue + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-vector\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + vv := htmlquery.InnerText(nodes[0]) + vv = strings.Replace(vv, "(", "", -1) + vv = strings.Replace(vv, ")", "", -1) + xpathList.VectorValue = strings.TrimSpace(vv) + } + + //xpathList.AccessVector + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-av\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AccessVector = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.AccessComplexity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-ac\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AccessComplexity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Authentication + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-au\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Authentication = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Confidentiality + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-c\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Confidentiality = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Integrity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-i\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Integrity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Availability + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-a\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Availability = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + } + } + } + + return xpathList, nil +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go b/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go new file mode 100644 index 0000000000000000000000000000000000000000..80cfbf946aedb3a2e1e84c88f018de9cf9960728 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go @@ -0,0 +1,49 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/xuri/excelize/v2" +) + +func GenerateExcels(status, path string) { + results, rowsAffected, err := db_models.GetResults(status) + if err != nil { + logs.Error("db_models.GetResults error:", err) + return + } + if rowsAffected == 0 { + logs.Info("No issue record") + return + } + + f := excelize.NewFile() + index := f.NewSheet("Sheet1") + f.SetActiveSheet(index) + cc := []string{"id", "cve_id", "cve_num", "cve_desc", "cve_level", "cve_version", "repair_time", "pack_name", "nvd_score", "n_vector_value", "create_time", "update_time", "delete_time", "status", "error_description"} + rowBegin := "A1" + err = f.SetSheetRow("Sheet1", rowBegin, &cc) + if err != nil { + logs.Error("set row title error:", err) + return + } + + for k, v := range results { + rowBegin = fmt.Sprintf("A%d", k+2) + err := f.SetSheetRow("Sheet1", rowBegin, &[]interface{}{v.IssueRecordId, v.CveId, v.CveNum, v.Description, v.CveLevel, v.CveVersion, v.RepairTime, v.PackName, v.NVDScore, v.NvectorVule, v.CreateTime, v.UpdateTime, v.DeleteTime, v.Status, v.ErrorDescription}) + if err != nil { + logs.Error("f.SetSheetRow error:", err, rowBegin) + continue + } + } + err = f.SaveAs(path) + if err != nil { + logs.Error("f.SaveAs error: ", err, path) + } + + err = db_models.UpdateStatue(status) + if err != nil { + logs.Error("db_models.UpdateStatue error: ", err) + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..9b095e2ba0663036550eabc067bbcc9efb9faff0 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go @@ -0,0 +1,291 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "io" + "net/http" + "os" + "time" +) + +// DownloadGuessYaml Download the yaml file of openGauss on Gitee +func DownloadGuessYaml() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "gauss_yaml/" + err := util.MakeDir(filePath) + if err != nil { + return "", err + } + yamlFileName := filePath + "gauss_yaml.yaml" + //download the yaml file + downloadUrl := "https://gitee.com/opengauss/openGauss-third_party/raw/master/Third_Party_Open_Source_Software_List.yaml" + resp, err := http.Get(downloadUrl) + defer resp.Body.Close() + if err != nil { + logs.Error("get", downloadUrl, "error", err.Error()) + return "", err + } + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + return "", errors.New(resp.Status) + } + saveFile, err := os.Create(yamlFileName) + defer saveFile.Close() + if err != nil { + logs.Error("Create file error", yamlFileName, err.Error()) + return "", err + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy resp.Body to file error", yamlFileName, err.Error()) + return "", err + } + return yamlFileName, nil +} + +func StoreYamlData(yamlData map[string]map[string]string) { + if len(yamlData) > 0 { + err := db_models.UpdateYamlOriginMark() + if err != nil { + logs.Error("db_models.UpdateYamlOriginMark error:", err.Error()) + return + } + for k, v := range yamlData { + if _, ok := v["version"]; !ok || v["version"] == "" { + v["version"] = "" + } + //open transaction + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + originData, err := db_models.SelectYamlOriginData(k, v["version"]) + now := time.Now() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + logs.Warn("error is orm.ErrNoRows, Create a record") + openGuessYaml := &models.OpenGussYaml{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + CreateTime: now.Format("2006-01-02 15:04:05"), + UpdateTime: now.Format("2006-01-02 15:04:05"), + MarkBit: 1, + Repo: "security", + } + err = db_models.InsertYamlOriginData(openGuessYaml) + if err != nil { + logs.Error("db_models.InsertYamlOriginData error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + logs.Info("insert OpenGussYaml success") + } else { + logs.Error("db_models.SelectYamlOriginData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("database records, change the data") + openGuessYaml := &models.OpenGussYaml{ + Id: originData.Id, + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + UpdateTime: now.Format("2006-01-02 15:04:05"), + Repo: "security", + MarkBit: 1, + } + _, err = db_models.UpdateYamlOriginData(openGuessYaml) + if err != nil { + logs.Error("db_models.UpdateYamlOriginData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"]) + if err != nil { + logs.Error("db_models.SelectYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + if rowsAffected == 0 { + logs.Warn("No record found, Create a record") + var packId int64 + gitOpenEuler, err := db_models.SelectYamlLastData() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + packId = 10000000 + } else { + logs.Error("db_models.SelectYamlLastData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + if gitOpenEuler.PackageId >= 10000000 { + packId = gitOpenEuler.PackageId + 1 + } else { + packId = 10000000 + } + } + goe := &models.GitOpenEuler{ + PackageId: packId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + } + lastId, err := db_models.InsertYamlData(goe) + if err != nil { + logs.Error("db_models.InsertYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + GitId: lastId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + Decription: "", + Status: 0, + } + err = db_models.InsertYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.InsertYamlDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + } else { + if rowsAffected > 1 { + for _, pv := range packAgesData[1:] { + err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + err = db_models.DeleteYamlOpenEulerData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + goe := &models.GitOpenEuler{ + OriginUrl: v["url"], + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + PackageId: packAgesData[0].PackageId, + PackageName: k, + Version: v["version"], + } + err = db_models.UpdateYamlData(goe) + if err != nil { + logs.Error("db_models.UpdateYamlData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + UpdateTime: now, + Status: 0, + GitId: packAgesData[0].GitId, + } + err = db_models.UpdateYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.UpdateYamlDetailData", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + logs.Info("Transaction committed successfully", k) + } + err = db_models.DeleteYamlOriginMark() + if err != nil { + logs.Error("db_models.DeleteYamlOriginMark error:", err.Error()) + } + } else { + logs.Warning("yamlData Is empty, nothing can be done") + } +} + +// ProcGaussYaml +//1. Create a folder +//2. Download yaml +//3. Parse yaml +//4. Save in mysql +func ProcGaussYaml() { + yamlFileName, err := DownloadGuessYaml() + if err != nil { + logs.Error("DownloadGuessYaml error:", err.Error()) + return + } + logs.Info("The download yaml file address is:", yamlFileName) + + yamlData, err := util.ParseYaml(yamlFileName) + if err != nil { + logs.Error("util.ParseYaml error:", yamlFileName, err) + return + } + StoreYamlData(yamlData) + + DeleteYamlData() + +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go b/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go new file mode 100644 index 0000000000000000000000000000000000000000..ea56900817f51343b09dac318293c8a6ae9d51b1 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go @@ -0,0 +1,194 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "errors" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/xuri/excelize/v2" + "io" + "os" + "strconv" + "time" +) + +func ImportDataTypeTwo() { + filePath, err := downloadfiletask.HandleThree() + if err != nil { + logs.Error("downloadfiletask.HandleThree error:", err) + return + } + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + f, err := os.Open(filePath + fileName) + if err != nil { + continue + } + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + _ = os.Remove(filePath + fileName) + continue + } + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + now := time.Now().Format("2006-01-02 15:04:05") + var packName string + var version string + var status int8 + var branchs string + for _, row := range rows[1:] { + if len(row) < 1 || row[0] == "" { + continue + } + packName = row[0] + if len(row) > 1 { + version = row[1] + if len(row) > 2 { + s, err := strconv.ParseInt(row[2], 10, 8) + if err != nil { + status = 1 + } else { + status = int8(s) + } + if len(row) > 3 { + branchs = row[3] + } else { + branchs = "" + } + } else { + status = 1 + } + } else { + version = "" + } + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + + _, err = db_models.SelectIssueRepoWhiteListByPackageName(row[0]) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + err = db_models.InsertIssueRepoWhiteList(&models.IssueRepoWhitelist{ + PackageName: packName, + Version: version, + Status: status, + Branchs: branchs, + CreateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertIssueRepoWhiteList:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Error("db_models.SelectIssueRepoWhiteListByPackageName error", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + err = db_models.UpdateIssueRepoWhiteList(models.IssueRepoWhitelist{ + Status: status, + Branchs: branchs, + UpdateTime: now, + PackageName: packName, + Version: version, + }) + if err != nil { + logs.Error("db_models.UpdateIssueRepoWhiteList error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + _ = excel.Close() + _ = os.Remove(filePath + fileName) + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go new file mode 100644 index 0000000000000000000000000000000000000000..a3dd6a6af43cd41faf41aa9e01131ede1d146cf0 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go @@ -0,0 +1,211 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "github.com/xuri/excelize/v2" + "io" + "os" + "time" +) + +// ToMysql Data stored in the database +func ToMysql() { + filePath, err := downloadfiletask.HandleTwo() + if err != nil { + logs.Error("downloadfiletask.HandleTwo error:", err) + return + } + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + f, err := os.Open(filePath + fileName) + if err != nil { + continue + } + + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + + _, err = os.Stat(beego.AppConfig.String("timedtask::packageDir") + "oldexcels/") + if err != nil { + logs.Error("os.Stat error:", err.Error()) + if os.IsNotExist(err) { + logs.Info("And there is no such directory", filePath, "start to create") + err = os.Mkdir(beego.AppConfig.String("timedtask::packageDir")+"oldexcels/", os.ModePerm) + if err != nil { + logs.Error("os.Mkdir error:", filePath, err.Error()) + } + logs.Info("create directory success:", filePath) + } + } + err = os.Rename(filePath+fileName, beego.AppConfig.String("timedtask::packageDir")+"oldexcels/"+fileName) + if err != nil { + logs.Error(err.Error()) + } + continue + } + + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err, filePath+fileName) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + now := time.Now().Format("2006-01-02 15:04:05") + var packName string + var cpePackName string + for _, row := range rows[1:] { + if len(row) < 1 || row[0] == "" { + continue + } + packName = row[0] + + if len(row) > 1 { + cpePackName = row[1] + } else { + cpePackName = "" + } + + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + packageCpe, err := db_models.SelectCvePackAgeCpeByPackName(packName) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + err := db_models.InsertCvePackAgeCpe(&models.PackageCpe{ + PackName: packName, + CpePackName: cpePackName, + CreateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertCvePackAgeCpe error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Error("db_models.SelectCvePackAgeCpeByPackName error: ", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + if packageCpe.CpePackName != cpePackName { + err := db_models.UpdateCvePackAgeCpe(models.PackageCpe{ + CpePackName: cpePackName, + CreateTime: now, + PackName: packageCpe.PackName, + }) + if err != nil { + logs.Error("db_models.UpdateCvePackAgeCpe error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + _, err = os.Stat(beego.AppConfig.String("timedtask::packageDir") + "oldexcels/") + if err != nil { + logs.Error("os.Stat error:", err.Error()) + if os.IsNotExist(err) { + logs.Info("And there is no such directory", filePath, "start to create") + err = os.Mkdir(beego.AppConfig.String("timedtask::packageDir")+"oldexcels/", os.ModePerm) + if err != nil { + logs.Error("os.Mkdir error:", filePath, err.Error()) + } + logs.Info("create directory success:", filePath) + } + } + err = os.Rename(filePath+fileName, beego.AppConfig.String("timedtask::packageDir")+"oldexcels/"+fileName) + if err != nil { + logs.Error(err.Error()) + } + continue + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..4bbe08bd30e0f1f7ffcf3de83b929c24c10e48ac --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go @@ -0,0 +1,292 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "io" + "net/http" + "os" + "time" +) + +// DownloadGuessYaml Download the yaml file of openGauss on Gitee +func DownloadMindSporeYaml() (string, error) { + filePath := beego.AppConfig.String("timedtask::packageDir") + "mindspore_yaml/" + err := util.MakeDir(filePath) + if err != nil { + return "", err + } + yamlFileName := filePath + "mindspore_yaml.yaml" + //download the yaml file + downloadUrl := "https://gitee.com/mindspore/community/raw/master/security/config/Third_Party_Open_Source_Software_List.yaml" + resp, err := http.Get(downloadUrl) + defer resp.Body.Close() + if err != nil { + logs.Error("get", downloadUrl, "error", err.Error()) + return "", err + } + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + return "", errors.New(resp.Status) + } + saveFile, err := os.Create(yamlFileName) + defer saveFile.Close() + if err != nil { + logs.Error("Create file error", yamlFileName, err.Error()) + return "", err + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy resp.Body to file error", yamlFileName, err.Error()) + return "", err + } + return yamlFileName, nil +} + +// StoreOpenLookengYaml parse dict +//Store yaml data to mysql +func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { + if len(yamlData) > 0 { + err := db_models.UpdateMindYamlOriginMark() + if err != nil { + logs.Error("db_models.UpdateYamlOriginMarkLookeng:", err.Error()) + return + } + for RepoKey, RepoValue := range yamlData { + for k, v := range RepoValue { + if _, ok := v["version"]; !ok || v["version"] == "" { + v["version"] = "" + } + //open transaction + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + originData, err := db_models.SelectMindYamlOriginData(k, v["version"], RepoKey) + now := time.Now() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + logs.Warn("error is orm.ErrNoRows, Create a record") + mindSporeYaml := &models.MindSporeYaml{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + CreateTime: now.Format("2006-01-02 15:04:05"), + UpdateTime: now.Format("2006-01-02 15:04:05"), + MarkBit: 1, + Repo: RepoKey, + } + err = db_models.InsertMindYamlOriginData(mindSporeYaml) + if err != nil { + logs.Error("db_models.InsertMindYamlOriginData error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + logs.Info("insert MindSporeYaml success") + } else { + logs.Error("db_models.SelectMindYamlOriginData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("database records, change the data") + mindSporeYaml := &models.MindSporeYaml{ + Id: originData.Id, + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + UpdateTime: now.Format("2006-01-02 15:04:05"), + MarkBit: 1, + } + _, err = db_models.UpdateMindYamlOriginData(mindSporeYaml) + if err != nil { + logs.Error("db_models.UpdateMindYamlOriginData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"]) + if err != nil { + logs.Error("db_models.SelectMindYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + if rowsAffected == 0 { + logs.Warn("No record found, Create a record") + var packId int64 + gitOpenEuler, err := db_models.SelectYamlLastData() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + packId = 10000000 + } else { + logs.Error("db_models.SelectYamlLastData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + if gitOpenEuler.PackageId >= 10000000 { + packId = gitOpenEuler.PackageId + 1 + } else { + packId = 10000000 + } + } + goe := &models.GitOpenEuler{ + PackageId: packId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + } + lastId, err := models.InsertYamlData(goe) + if err != nil { + logs.Error("db_models.InsertYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + GitId: lastId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + Decription: "", + Status: 0, + } + err = db_models.InsertYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.InsertYamlDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + } else { + if rowsAffected > 1 { + for _, pv := range packAgesData[1:] { + err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + err = db_models.DeleteYamlOpenEulerData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + goe := &models.GitOpenEuler{ + OriginUrl: v["url"], + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + PackageId: packAgesData[0].PackageId, + PackageName: k, + Version: v["version"], + } + err = models.UpdateYamlData(goe) + if err != nil { + logs.Error("db_models.UpdateYamlData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + UpdateTime: now, + Status: 0, + GitId: packAgesData[0].GitId, + } + err = db_models.UpdateYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.UpdateYamlDetailData", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + logs.Info("Transaction committed successfully", k) + } + } + err = db_models.DeleteMindYamlOriginMark() + if err != nil { + logs.Error("db_models.DeleteMindYamlOriginMark error:", err.Error()) + } + } else { + logs.Warning("yamlData Is empty, nothing can be done") + } + +} + +// ProcOpenLookengYaml +//1. Create a folder; +//2. Download yaml; +//3. Parse yaml; +//4. Save in mysql; +func ProcMindSporeYaml() { + yamlFileName, err := DownloadMindSporeYaml() + if err != nil { + logs.Error("DownloadMindSporeYaml error:", err.Error()) + return + } + yamlData, err := util.ParseYamlTypeTwo(yamlFileName) + if err != nil { + logs.Error("util.ParseYamlTypeTwo error: ", yamlFileName, err) + return + } + StoreMindSporeYaml(yamlData) + + DeleteYamlData() + +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go new file mode 100644 index 0000000000000000000000000000000000000000..bc24825bbdb8d83d34cba05fb6e0a76b0bf3adef --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go @@ -0,0 +1,132 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "github.com/astaxie/beego/logs" + "github.com/xuri/excelize/v2" + "io" + "os" + "time" +) + +// import excel +func ImportData() { + filePath, err := downloadfiletask.HandleFour() + if err != nil { + logs.Error("downloadfiletask.HandleFour error:", err) + return + } + + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + f, err := os.Open(filePath + fileName) + if err != nil { + continue + } + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + _ = os.Remove(filePath + fileName) + continue + } + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + _ = excel.Close() + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + now := time.Now().Format("2006-01-02 15:04:05") + var packName string + var assignee string + for _, row := range rows[1:] { + if len(row) < 1 || row[0] == "" { + continue + } + packName = row[0] + + if len(row) > 1 { + assignee = row[1] + } else { + assignee = "" + } + err = db_models.InsertSpecIssueAssigness(&models.SpecIssueAssigness{ + PackageName: packName, + Assignee: assignee, + Status: 1, + CreateTime: now, + }) + if err != nil { + logs.Error(err) + continue + } + } + _ = excel.Close() + _ = os.Remove(filePath + fileName) + } + +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go new file mode 100644 index 0000000000000000000000000000000000000000..35ce2f21db33f88e4dc310cb5a81d4a26e43cc4c --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go @@ -0,0 +1,93 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/models" + "fmt" + "github.com/astaxie/beego/logs" + "strconv" + "time" +) + +// RepeatTask Parse the artificial CVE table, crawl the CVE official +// website data, and store it in the database +func RepeatTask() { + logs.Info("CVE with empty CVE official website data is re-crawled ") + + originExcel, rowsAffected, err := db_models.SelectCveOriginExcel("") + if err != nil { + logs.Error("db_models.SelectCveOriginExcel error:", err) + return + } + if rowsAffected == 0 { + return + } + now := time.Now().Format("2006-01-02 15:04:05") + for _, v := range originExcel { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", v.CveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err, url) + continue + } + nvdScore, err := strconv.ParseFloat(crawlList.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + err = db_models.UpdateCveOriginExcelTypeFour(models.OriginExcel{ + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 1, + UpdateTime: now, + ScoreType: crawlList.ScoreType, + CveNum: v.CveNum, + PackName: v.PackName, + CveVersion: v.CveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeFour error:", err) + } + } +} + +// GetPublishedDateTask Get the release date of nvd +func GetPublishedDateTask() { + logs.Info("Get the release date of nvd") + vulnCenter, rowsAffected, err := db_models.SelectCveVulnCenter() + if err != nil { + logs.Error("db_models.SelectCveVulnCenter error:", err) + return + } + if rowsAffected == 0 { + return + } + for _, v := range vulnCenter { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", v.CveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err, url) + continue + } + if crawlList.RepairTime != "" { + err = db_models.UpdateCveVulnCenter(crawlList.RepairTime, strconv.FormatInt(v.CveId, 10)) + if err != nil { + logs.Error("db_models.UpdateCveVulnCenter error:", err) + continue + } + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/run_task.go b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go new file mode 100644 index 0000000000000000000000000000000000000000..156b62c2662b4bddd0198a5ef3d193420ad2501a --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go @@ -0,0 +1,317 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "github.com/xuri/excelize/v2" + "io" + "os" + "strconv" + "time" +) + +// HandleDate CVSS official website data crawling data storage database +func HandleDate() { + filePath, err := downloadfiletask.HandleOne() + if err != nil { + logs.Error("downloadfiletask.HandleOne error:", err) + return + } + + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + + f, err := os.Open(filePath + fileName) + if err != nil { + continue + } + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + fmt.Println(hashValue) + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + _ = os.Remove(filePath + fileName) + continue + } + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + + var packName string + var cveVersion string + var cveNum string + now := time.Now().Format("2006-01-02 15:04:05") + for _, row := range rows[1:] { + if len(row) < 1 { + continue + } + packName = row[0] + if len(row) > 1 { + cveVersion = row[1] + if len(row) > 2 { + cveNum = row[2] + } else { + cveNum = "" + } + } else { + cveVersion = "" + } + + logs.Info("begin transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + resultDict, err := db_models.GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + nvdScore, err := strconv.ParseFloat(crawlList.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + err = db_models.InsertCveOriginExcel(&models.OriginExcel{ + CveNum: cveNum, + CveUrl: url, + CveVersion: cveVersion, + PackName: packName, + ScoreType: crawlList.ScoreType, + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 0, + CreateTime: now, + UpdateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertCveOriginExcel error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + + } else { + logs.Error("db_models.GetCveOriginExcelTypeTwo error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + if crawlList.NvdScore != "" || crawlList.CveDesc != "" { + nvdScore, err := strconv.ParseFloat(crawlList.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + if crawlList.CveDesc != "" && resultDict.NVDScore == nvdScore && resultDict.NVDVector == crawlList.VectorValue { + comp := make([]interface{}, 0, 10) + comp = append(comp, 3, 4, 7) + if util.InSlice(comp, int(resultDict.CveStatus)) { + logs.Info("update data aa:", cveNum) + err = db_models.UpdateCveOriginExcelTypeThree(models.OriginExcel{ + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 1, + UpdateTime: now, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeThree error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("The database is the latest data:", cveNum) + } + } else { + logs.Info("update data dd:", cveNum) + err = db_models.UpdateCveOriginExcelTypeFour(models.OriginExcel{ + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 1, + UpdateTime: now, + ScoreType: crawlList.ScoreType, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeFour error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } else { + logs.Error("error:", resultDict) + } + } + + specError, err := db_models.GetCveSpecError(cveNum) + if err != nil { + if !errors.Is(orm.ErrNoRows, err) { + logs.Error("db_models.GetCveSpecError error:", err) + } + } else { + err = db_models.UpdateCveOriginExcelTypeFive(models.OriginExcel{ + CveDesc: specError.Description, + CveStatus: 6, + CveNum: cveNum, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeFive error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go new file mode 100644 index 0000000000000000000000000000000000000000..6f25f90a63c567995217c042354f86b5bff2a251 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go @@ -0,0 +1,136 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/models" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "github.com/xuri/excelize/v2" + "os" + "time" +) + +// AddErrorDetails Conditions for filtering incorrect data +func AddErrorDetails() { + filePath, err := downloadfiletask.DownloadSpecErrorExcels() + if err != nil { + logs.Error("downloadfiletask.DownloadSpecErrorExcels error:", err) + return + } + + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + var cveNum string + var cveDesc string + var cveOwner string + var packName string + now := time.Now().Format("2006-01-02 15:04:05") + for _, row := range rows[1:] { + if len(row) < 2 { + continue + } + cveNum = row[1] + if len(row) > 2 { + cveOwner = row[2] + if len(row) > 3 { + packName = row[3] + if len(row) > 4 { + cveDesc = row[4] + } else { + cveDesc = "" + } + } else { + packName = "" + } + } else { + cveOwner = "" + } + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + specError, err := db_models.GetCveSpecError(cveNum) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + _, err = db_models.InsertCveSpecError(&models.SpecError{ + CveNum: cveNum, + Description: cveDesc, + Owner: cveOwner, + Status: 1, + PackName: packName, + CreateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertCveSpecError:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Error("db_models.GetCveSpecError:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("CVE recorded", cveNum) + if specError.Description != cveDesc || specError.Owner != cveOwner || specError.PackName != packName || specError.Status != 1 { + logs.Info("update cve_spec_error") + err := db_models.UpdateCveSpecError(models.SpecError{ + CveNum: specError.CveNum, + Description: cveDesc, + Owner: cveOwner, + PackName: packName, + Status: 1, + UpdateTime: now, + }) + if err != nil { + logs.Error("db_models.UpdateCveSpecError:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + _ = os.Remove(filePath + fileName) + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go new file mode 100644 index 0000000000000000000000000000000000000000..e436b0c0b428be7d0b9b804bfa91657a4f9cd8be --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go @@ -0,0 +1,391 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +//var cveListDate []db_models.VulnCenter + +type Compare struct { + CveId string + CveNum string +} + +// QueryCveALLData Find cve with missing necessary fields +func QueryCveALLData(days int) []models.VulnCenter { + now := time.Now() + beforeDate := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, now.Location()).Format("2006-01-02 15:04:05") + + cveIdList := make([]Compare, 0, 100) + scoreList, rowsAffected, err := db_models.GetCveScoreListByBeforeDate(beforeDate) + if err == nil { + if rowsAffected > 0 { + for _, v := range scoreList { + cveIdList = append(cveIdList, Compare{ + CveId: strconv.FormatInt(v.CveId, 10), + CveNum: v.CveNum, + }) + } + } else { + logs.Info("No query results") + } + } else { + logs.Error("db_models.GetCveScoreList error:", err.Error()) + } + vulnCenterList, rowsAffected, err := db_models.GetCveVulnCenterList("", "", beforeDate) + if err == nil { + if rowsAffected > 0 { + for _, v := range vulnCenterList { + cveIdList = append(cveIdList, Compare{ + CveId: strconv.FormatInt(v.CveId, 10), + CveNum: v.CveNum, + }) + } + } else { + logs.Info("No query results") + } + } else { + logs.Error("db_models.GetCveVulnCenter error:", err.Error()) + } + + issueTemplateList, rowsAffected, err := db_models.GetCveIssueTemplate("0", "", "", beforeDate) + if err == nil { + if rowsAffected > 0 { + for _, v := range issueTemplateList { + cveIdList = append(cveIdList, Compare{ + CveId: strconv.FormatInt(v.CveId, 10), + CveNum: v.CveNum, + }) + } + } else { + logs.Info("No query results") + } + + } else { + logs.Error("db_models.GetCveIssueTemplate error:", err.Error()) + } + + cveIdList = RemoveRepeatedElement(cveIdList) + cveList := make([]models.VulnCenter, 0, len(cveIdList)) + if len(cveIdList) > 0 { + for _, v := range cveIdList { + vulnCenter, err := db_models.GetOneVulnCenter(v.CveId, v.CveNum) + if err != nil { + logs.Error("db_models.GetOneVulnCenter error:", err.Error()) + continue + } + + cveList = append(cveList, vulnCenter) + } + } + return cveList +} + +func RemoveRepeatedElement(arr []Compare) (newArr []Compare) { + newArr = make([]Compare, 0) + for i := 0; i < len(arr); i++ { + repeat := false + for j := i + 1; j < len(arr); j++ { + if arr[i] == arr[j] { + repeat = true + break + } + } + if !repeat { + newArr = append(newArr, arr[i]) + } + } + return +} + +// UpdateCveVuln update data +func UpdateCveVuln(url string, cve models.VulnCenter) error { + cveNum := strings.TrimSpace(cve.CveNum) + cveId := cve.CveId + cveStatus := cve.Status + cveDesc := strings.TrimSpace(cve.Description) + repairTime := strings.TrimSpace(cve.RepairTime) + updateTime := time.Now() + + cveStatusx := cveStatus + comp := make([]interface{}, 0, 10) + comp = append(comp, 0, 3, 4, 5, 6, 7, 8) + if util.InSlice(comp, int(cveStatus)) { + cveStatusx = 0 + } + compTwo := make([]interface{}, 0, 10) + compTwo = append(compTwo, 1, 2) + if util.InSlice(compTwo, int(cveStatus)) { + cveStatusx = 1 + } + listx, err := Crawling(url) + if err != nil { + logs.Error("crawling data error:", err, url) + return err + } + + if (listx.CveDesc == "" || len(listx.CveDesc) < 2) && (listx.NvdScore == "") && (listx.RepairTime == "" || len(listx.RepairTime) > 2) { + logs.Info("The data does not exist and will not be processed temporarily", listx) + return errors.New("The data does not exist and will not be processed temporarily") + } + + if repairTime == "" && listx.RepairTime != "" && len(listx.RepairTime) > 2 { + if len(listx.RepairTime) > 10 { + repairTime = listx.RepairTime[:10] + } else { + repairTime = listx.RepairTime + } + err = db_models.UpdateVulnCenterTypeOne(models.VulnCenter{ + RepairTime: repairTime, + UpdateTime: updateTime, + Status: cveStatusx, + CveId: cveId, + }) + if err != nil { + logs.Error("db_models.UpdateVulnCenterTypeOne error:", err) + return err + } + } + + if listx.CveDesc != "" && len(listx.CveDesc) > 2 && cveDesc == "" { + err = db_models.UpdateVulnCenterTypeTwo(models.VulnCenter{ + Description: listx.CveDesc, + UpdateTime: updateTime, + Status: cveStatusx, + CveId: cveId, + }) + if err != nil { + logs.Error("db_models.UpdateVulnCenterTypeTwo error:", err) + return err + } + } + + nvdScore, err := strconv.ParseFloat(listx.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + if nvdScore > 0 && listx.NvdScore != "" { + var scoreType string + if listx.ScoreType == "v3.0" { + scoreType = "v3" + } else { + scoreType = "v2" + } + score, err := db_models.GetCveScoreByCveIdAndCveNum(cveId, cveNum) + if err != nil { + if errors.Is(err, orm.ErrNoRows) { + logs.Info(err) + } else { + logs.Error("db_models.GetCveScoreByCveIdAndCveNum error:", err) + return err + } + } else { + if score.NVDScore == 0 { + err := db_models.UpdateCveScore(models.Score{ + NVDScore: nvdScore, + NvectorVule: listx.VectorValue, + NattackVector: listx.AttackVector, + NaccessVector: listx.AccessVector, + NattackComplexity: listx.AttackComplexity, + NaccessComplexity: listx.AccessComplexity, + NprivilegeRequired: listx.PrivilegeRequired, + NuserInteraction: listx.UserInteraction, + Nscope: listx.Scope, + Nconfidentiality: listx.Confidentiality, + Nintegrity: listx.Integrity, + Navailability: listx.Availability, + Nauthentication: listx.Authentication, + UpdateTime: updateTime, + ScoreType: scoreType, + Id: score.Id, + }) + if err != nil { + logs.Error("db_models.UpdateCveScore error:", err) + return err + } + } + } + } + + templateResult, err := db_models.GetIssueTemplateTypeOne(models.IssueTemplate{ + NVDScore: 0, + NVDVector: "", + CveBrief: "", + CveId: cveId, + CveNum: cveNum, + }) + if err != nil { + logs.Error(" db_models.GetIssueTemplateTypeOne error:", err) + return nil + } + + flag := false + if templateResult.NVDScore == 0 && listx.NvdScore != "" && nvdScore > 0 { + err = db_models.UpdateCveIssueTemplateTypeOne(models.IssueTemplate{ + NVDScore: nvdScore, + UpdateTime: updateTime, + TemplateId: templateResult.TemplateId, + }) + if err != nil { + logs.Error("db_models.UpdateCveIssueTemplateTypeOne error:", err) + return err + } + flag = true + } + + if (templateResult.NVDVector == "" || len(templateResult.NVDVector) < 2) && (listx.VectorValue != "" && len(listx.VectorValue) > 2) { + err = db_models.UpdateCveIssueTemplateTypeTwo(models.IssueTemplate{ + NVDVector: listx.VectorValue, + UpdateTime: updateTime, + TemplateId: templateResult.TemplateId, + }) + if err != nil { + logs.Error("db_models.UpdateCveIssueTemplateTypeTwo error:", err) + return err + } + flag = true + } + + if (templateResult.CveBrief == "" || len(templateResult.CveBrief) < 2) && (listx.CveDesc != "" && len(listx.CveDesc) > 2) { + err = db_models.UpdateCveIssueTemplateTypeThree(models.IssueTemplate{ + CveBrief: listx.CveDesc, + UpdateTime: updateTime, + TemplateId: templateResult.TemplateId, + }) + if err != nil { + logs.Error("db_models.UpdateCveIssueTemplateTypeThree error:", err) + return err + } + flag = true + } + + if flag { + err = db_models.UpdateVulnCenter(models.VulnCenter{ + UpdateTime: updateTime, + Status: cveStatusx, + CveId: cveId, + }) + if err != nil { + logs.Error("db_models.UpdateVulnCenter error:", err) + return err + } + } + return nil +} + +// SupplementCve +//1. Find cve with missing necessary fields; +//2. Go to the CVE official website to find the CVE information; +//3. Fill in the corresponding table again; +func SupplementCve() { + + cveList := QueryCveALLData(30) + + if len(cveList) > 0 { + for _, cve := range cveList { + cveNum := strings.TrimSpace(cve.CveNum) + cveVersion := strings.TrimSpace(cve.CveVersion) + packName := strings.TrimSpace(cve.PackName) + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + _ = UpdateCveVuln(url, cve) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + if err != nil { + logs.Error("db_models.GetSpceError error:", err) + continue + } + logs.Info("Filter, modify the status to 6", cveNum) + err = db_models.UpdateCveOriginExcel(models.OriginExcel{ + CveDesc: resultSpecError.Description, + CveStatus: 6, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcel error:", err) + } + } + } +} + +// LongSupplementCve 1. Find cve with missing necessary fields; +//2. Go to the CVE official website to find the CVE information; +//3. Fill in the corresponding table again; +func LongSupplementCve() { + cveList := QueryCveALLData(1500) + if len(cveList) > 0 { + for _, cve := range cveList { + cveNum := strings.TrimSpace(cve.CveNum) + cveVersion := strings.TrimSpace(cve.CveVersion) + packName := strings.TrimSpace(cve.PackName) + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + _ = UpdateCveVuln(url, cve) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + if err != nil { + logs.Error("db_models.GetSpceError error:", err) + continue + } + logs.Info("Filter, modify the status to 6", cveNum) + err = db_models.UpdateCveOriginExcel(models.OriginExcel{ + CveDesc: resultSpecError.Description, + CveStatus: 6, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcel error:", err) + } + } + } +} + +func PullCve(cveNum string) (resp int) { + resp = 0 + cveList, rowsAffected, err := db_models.QueryCveByNumber(cveNum) + if err != nil { + return resp + } + if rowsAffected == 0 { + return resp + } + fmt.Println(rowsAffected) + + if len(cveList) > 0 { + for _, cve := range cveList { + cveNum := strings.TrimSpace(cve.CveNum) + cveVersion := strings.TrimSpace(cve.CveVersion) + packName := strings.TrimSpace(cve.PackName) + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + err = UpdateCveVuln(url, cve) + if err == nil { + resp = 1 + } + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + if err != nil { + logs.Error("db_models.GetSpceError error:", err) + } + logs.Info("Filter, modify the status to 6", cveNum) + err = db_models.UpdateCveOriginExcel(models.OriginExcel{ + CveDesc: resultSpecError.Description, + CveStatus: 6, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcel error:", err) + } + } + } + return resp +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go b/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go new file mode 100644 index 0000000000000000000000000000000000000000..b3cc56e67d3d440f15c7f891c4cde77346a830be --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go @@ -0,0 +1,52 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/xuri/excelize/v2" +) + +func GenerateTable(path string) { + results, rowsAffected, err := db_models.GetCveOriginExcel() + if err != nil { + logs.Error("db_models.GetCveOriginExcel error:", err) + return + } + if rowsAffected == 0 { + logs.Info("The query result is empty, no error data") + return + } + + f := excelize.NewFile() + index := f.NewSheet("Sheet1") + f.SetActiveSheet(index) + + cc := []string{"cve_id", "cve_num", "cve_url", "cve_version", "pack_name", "score_type", "nvd_score", "cve_level", "cve_desc", "repair_time", "vector_value", "attack_vector", "access_vector", "attack_complexity", "access_complexity", "privilege_required", "user_interaction", "scope", "confidentiality", "integrity", "availability", "authentication", "cve_status", "create_time", "update_time", "delete_time"} + rowBegin := "A1" + err = f.SetSheetRow("Sheet1", rowBegin, &cc) + if err != nil { + logs.Error("set row title error:", err) + return + } + + for k, v := range results { + rowBegin = fmt.Sprintf("A%d", k+2) + err := f.SetSheetRow("Sheet1", rowBegin, &[]interface{}{v.CveId, v.CveNum, v.CveUrl, v.CveVersion, v.PackName, v.ScoreType, v.NVDScore, v.CveLevel, v.CveDesc, v.RepairTime, v.NVDVector, v.AttackVector, v.AccessVector, v.AttackComplexity, v.AccessComplexity, v.PrivilegeRequired, v.UserInteraction, v.Scope, v.Confidentiality, v.Integrity, v.Availability, v.Authentication, v.CveStatus, v.CreateTime, v.UpdateTime, v.UpdateTime}) + if err != nil { + logs.Error("f.SetSheetRow error:", err, rowBegin) + continue + } + } + + err = f.SaveAs(path) + if err != nil { + logs.Error("f.SaveAs error: ", err, path) + } + + err = db_models.UpdateStatusTypeTwo() + if err != nil { + logs.Error("db_models.UpdateStatusTypeTwo error: ", err) + } + +} diff --git a/cve-vulner-manager/cve-timed-task/util/crawl.go b/cve-vulner-manager/cve-timed-task/util/crawl.go new file mode 100644 index 0000000000000000000000000000000000000000..71af4ef9f9ad46fd3723bf642768643f3dbd37c7 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/crawl.go @@ -0,0 +1,28 @@ +package util + +import ( + "github.com/pkg/errors" + "golang.org/x/net/html" + "golang.org/x/net/html/charset" + "net/http" +) + +func UrlToHTML(url string) (*html.Node, error) { + resp, err := http.Get(url) + defer resp.Body.Close() + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, errors.New(resp.Status) + } + r, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) + if err != nil { + return nil, err + } + doc, err := html.Parse(r) + if err != nil { + return nil, err + } + return doc, nil +} diff --git a/cve-vulner-manager/cve-timed-task/util/deduplication.go b/cve-vulner-manager/cve-timed-task/util/deduplication.go new file mode 100644 index 0000000000000000000000000000000000000000..b8ab057341f621f2636590c5dd63901f8f73f4a9 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/deduplication.go @@ -0,0 +1,10 @@ +package util + +func InSlice(list []interface{}, item interface{}) bool { + for _, eachItem := range list { + if item == eachItem { + return true + } + } + return false +} diff --git a/cve-vulner-manager/cve-timed-task/util/makrdir.go b/cve-vulner-manager/cve-timed-task/util/makrdir.go new file mode 100644 index 0000000000000000000000000000000000000000..524c9ec20fe8223b3ebb5618c12946ae0bdd494e --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/makrdir.go @@ -0,0 +1,53 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" +) + +// MakeDir Create a directory,if there has this directory, delete all files in this directory +func MakeDir(filePath string) error { + _, err := os.Stat(filePath) + if err != nil { + logs.Error("os.Stat error:", err.Error()) + if os.IsNotExist(err) { + logs.Info("And there is no such directory", filePath, "start to create") + err = os.MkdirAll(filePath, os.ModePerm) + if err != nil { + logs.Error("os.MkdirAll error:", filePath, err.Error()) + return err + } + logs.Info("create directory success:", filePath) + return nil + } + return err + } + + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + return err + } + + for _, file := range fileList { + fileName := file.Name() + childFilePath := filePath + fileName + if file.IsDir() { + err = os.RemoveAll(childFilePath) + if err != nil { + logs.Error("os.RemoveAll error:", childFilePath, err.Error()) + return err + } + logs.Info("directory deleted successfully") + } else { + err = os.Remove(childFilePath) + if err != nil { + logs.Error("os.Remove error:", childFilePath, err.Error()) + return err + } + logs.Info("Delete file successfully") + } + } + return nil +} diff --git a/cve-vulner-manager/cve-timed-task/util/parse_yaml.go b/cve-vulner-manager/cve-timed-task/util/parse_yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..76951204e430c7b298e0f57567be3257c0722fdf --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/parse_yaml.go @@ -0,0 +1,42 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "gopkg.in/yaml.v2" + "io/ioutil" +) + +// ParseYaml Parse yaml +func ParseYaml(filePath string) (map[string]map[string]string, error) { + file, err := ioutil.ReadFile(filePath) + if err != nil { + logs.Error("ioutil.ReadFile error:", filePath, err.Error()) + return nil, err + } + + resultMap := make(map[string]map[string]string) + err = yaml.Unmarshal(file, resultMap) + if err != nil { + logs.Error("yaml.Unmarshal error:", err) + return nil, err + } + + return resultMap, nil +} + +func ParseYamlTypeTwo(filePath string) (map[string]map[string]map[string]string, error) { + file, err := ioutil.ReadFile(filePath) + if err != nil { + logs.Error("ioutil.ReadFile error:", filePath, err.Error()) + return nil, err + } + resultMap := make(map[string]map[string]map[string]string) + + err = yaml.Unmarshal(file, resultMap) + if err != nil { + logs.Error("yaml.Unmarshal error:", err) + return nil, err + } + return resultMap, nil + +} diff --git a/cve-vulner-manager/doc/excel/packagewhitelist.xlsx b/cve-vulner-manager/doc/excel/packagewhitelist.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1fc7a53c3b41641c9f00853377a5611a6d273ff2 Binary files /dev/null and b/cve-vulner-manager/doc/excel/packagewhitelist.xlsx differ diff --git a/cve-vulner-manager/doc/image/Framework.png b/cve-vulner-manager/doc/image/Framework.png new file mode 100644 index 0000000000000000000000000000000000000000..eacb23ff606557a026aff12b560ef60dd69718c0 Binary files /dev/null and b/cve-vulner-manager/doc/image/Framework.png differ diff --git "a/cve-vulner-manager/doc/image/\350\247\246\345\217\221issue\346\265\201\347\250\213\345\233\276.png" "b/cve-vulner-manager/doc/image/\350\247\246\345\217\221issue\346\265\201\347\250\213\345\233\276.png" new file mode 100644 index 0000000000000000000000000000000000000000..beb34a870166556336f353cd638c3d5435249967 Binary files /dev/null and "b/cve-vulner-manager/doc/image/\350\247\246\345\217\221issue\346\265\201\347\250\213\345\233\276.png" differ diff --git a/cve-vulner-manager/doc/md/installx.md b/cve-vulner-manager/doc/md/installx.md new file mode 100644 index 0000000000000000000000000000000000000000..2229e01d7dc04d8097e7c376016bc3c3051b77ab --- /dev/null +++ b/cve-vulner-manager/doc/md/installx.md @@ -0,0 +1,24 @@ +# Getting Started On Md + +## Build + + +* Before you get started, make sure to have [Go](https://golang.org/) already installed in your local machine. + +* Install beego + ```cassandraql + go get -u github.com/astaxie/beego + go get -u github.com/beego/bee + ``` + [Reference documents](https://beego.me/) + + * The configuration environment variable values are as follows: + ```cassandraql + DB_PWD,DB_USER,AES_KEY,GITEE_CLIENT_ID,GITEE_CLIENT_SECRET,GITEE_PASSWORD,HOOK_PWD,GITEE_TOKEN + ``` +## Usage + +* run + ``` + bee run + ``` \ No newline at end of file diff --git a/cve-vulner-manager/doc/md/manual.md b/cve-vulner-manager/doc/md/manual.md new file mode 100644 index 0000000000000000000000000000000000000000..2c4209eadfed234df8459e6d80914323244f302d --- /dev/null +++ b/cve-vulner-manager/doc/md/manual.md @@ -0,0 +1,121 @@ +## issue分析说明 + +### issue分析触发流程图 + + +#### issue 分析评论流程 + +issue分析注意事项 +- 所有人都可以在评论区对issue进行分析 +- 分析issue模板并拷贝以下内容在**评论区回复**： + ```batch + 影响性分析说明: + + openEuler评分: (评分和向量) + + 受影响版本排查(受影响/不受影响): + 1.openEuler-20.03-LTS(1.16.1): + 2.openEuler-20.03-LTS-SP1(1.16.1): + + 修复是否涉及abi变化（是/否）： + 1.openEuler-20.03-LTS(1.16.1): + 2.openEuler-20.03-LTS-SP1(1.16.1): + ``` +- 受影响版本排查(受影响/不受影响)填写说明(未引入分支不需要填写)： + ```batch + 1>issue模板或第一条评论中不包含 如：1.openEuler-20.03-LTS:之类的填写项则不需要填写， + 如Maintainer发现有对应的版本需要填写则参考下方说明格式填写 + 2>填写方式： + 1.openEuler-20.03-LTS(1.16.1):受影响 + 2.openEuler-20.03-LTS-SP1(1.16.1):不受影响 + ``` +- 修复是否涉及abi变化（是/否）填写说明(未引入分支不需要填写)： + ```batch + 1>issue模板或第一条评论中不包含 如：1.openEuler-20.03-LTS:之类的填写项则不需要填写， + 如Maintainer发现有对应的版本需要填写则参考下方说明格式填写 + 2>填写方式： + 1.openEuler-20.03-LTS(1.16.1):是 + 2.openEuler-20.03-LTS-SP1(1.16.1):否 + ``` + - 分析内容可一次性填写完整也可分条填写 + - 重复分析内容可覆盖上一次分析的内容（只要包含键值的评论不管内容是否为空都会覆盖） + - cve-manage 在每次分析评论提交后根据内容进行解析并根据解析后的内容进行回复，Maintainer在评论后没得到回复可主动刷新页面或有延迟 + - 当内容填写完整会触发两种回复中的一种： + + 1.评分不一致需要审核 + + | 状态 | 需分析 | 内容 | + |-----|----------------|------------------------| + | 已分析 | openEulerScore | 6.0 | + | 已分析 | 影响性分析说明 | 您分析的内容 | + | 已分析 | openEulerVector | AV:A/AC:C... | + | 已分析 | 受影响的版本 | openEuler-20.03-LTS:受影响 | + | 已分析 | 修复是否涉及abi变化 | openEuler-20.03-LTS:是 | + **因openEulerScore与NvdScore不一致，分析内容需审核，请等待审核** + + 2.评分一致 + + | 状态 | 需分析 | 内容 | + |-----|----------------|------------------------| + | 已分析 | openEulerScore | 6.0 | + | 已分析 | 影响性分析说明 | 您分析的内容 | + | 已分析 | openEulerVector | AV:A/AC:C... | + | 已分析 | 受影响的版本 | openEuler-20.03-LTS:受影响 | + | 已分析 | 修复是否涉及abi变化 | openEuler-20.03-LTS:是 | + **请确认分析内容的准确性，确认无误后，您可以进行后续步骤或您可以继续分析。** + - 评分不一致则需等待审核员审核后进行下一步 **否则将视为无效分析将不可导出** + - issue 已完成（关闭）且已分析完结、评分不一致需审核通过后的issue才具备发布安全公告到官网 + + ### 审核员操作说明 + - 安全委员会成员对填写的评分在当前issue的评论区回复如下指令 + - 使用指令说明 + + | 指令 | 指令说明 | 使用权限 | + | ------ | ------- | ------- | + | /approve | 审核通过 | security-committee-sig组成员 | + | /reject | 审核拒绝 | security-committee-sig组成员 | + + - 注意事项 + ```batch + 1.指令独占一行 + 2.以最后一位审核员评论审核的指令为最终指令 + 3.审核评论成功 cve-manage 会新增一条@责任人的评论 + （若没有新增评论，可主动刷新页面，多次刷新后没有回复则没有审核评论成功） +``` + ### issue评论区使用/error 指令说明 + - 当提交issue的cve漏洞信息出现错误时，处理当前issue的操作员可以选择将issue状态改为已拒绝， + - 操作步骤如下： + + 1.在issue评论区填写cve误报或者拒绝原因，格式如下： + + | 指令 | 指令说明 | 使用权限 | + | ------ | ------- | ------- | + | /error | /error +错误原因说明 | 不限 | + + 例如：/error cve误报 + + 2.将issue状态改为已拒绝 + + + ### 模板内容审核并关闭issue + - issue处理人需要审核模板内容是否填写正确，在当前issue的评论区回复如下指令 + - 使用指令说明 + + | 指令 | 指令说明 | 使用权限 | + | ------ | ------- | ------- | + | /approve | 模板内容审核并关闭issue | 具有处理issue权限的人 | + + ### 关闭issue + - 具有当前issue处理权限的人，可以通过如下指令关闭issue + - 使用指令说明 + + | 指令 | 指令说明 | 使用权限 | + | ------ | ------- | ------- | + | /close | 关闭issue(功能等同/approve) | 具有处理issue权限的人 | + +### 数据导出 +#### 1.调用数据导出接口，触发生成cvrf格式xml文档的任务 + - 接口说明： + 内部公开 + + diff --git a/cve-vulner-manager/doc/md/packwhiteapi.md b/cve-vulner-manager/doc/md/packwhiteapi.md new file mode 100644 index 0000000000000000000000000000000000000000..06b0b49e0e7888fd62a04d3f5bcec847ce1b4e28 --- /dev/null +++ b/cve-vulner-manager/doc/md/packwhiteapi.md @@ -0,0 +1,46 @@ +#### 1.导入指定仓库创建issue的excel文档模板 + * [excel模板链接](../excel/packagewhitelist.xlsx) +#### 2.仓库创建issue白名单查询接口 + - 接口说明： + +| 说明 | 内容 | +| --- | ---- | +| 请求地址 | https://api.openeuler.org/cve-manager/v1/cve/issue/whitelist | +| 请求类型 | GET | +- 参数： + +| 参数 | 是否必传 | 类型 | 数据类型 | 参数说明 | +| --- | --- | --- | --- | --- | +| status | yes | query | int8 | 返回的数据状态：0：查询全部数据；1：查询正常数据；2：查询已删除数据 | + +- 响应返回： + - 数据返回示例： + ``` + { + "body": [ + { + "id": 1, + "packageName": "firefox", # 包名 + "version": "79.0", # 版本信息 + "branchs": "", # 分支信息 + "status": 1, # 1：正常状态；2：已删除状态 + "createTime": "2020-11-03 12:24:23", + "updateTime": "", + "deleteTime": "" + }, + { + "id": 2, + "packageName": "sudo", + "version": "1.8.27", + "branchs": "", + "status": 1, + "createTime": "2020-11-03 12:24:23", + "updateTime": "", + "deleteTime": "" + } + ], + "errmsg": "成功", + "errno": "200" + } + ``` + diff --git a/cve-vulner-manager/doc/sql/db_struct.sql b/cve-vulner-manager/doc/sql/db_struct.sql new file mode 100644 index 0000000000000000000000000000000000000000..b4f35238e45c908f90e18f1ecaa155610f2fc6b8 --- /dev/null +++ b/cve-vulner-manager/doc/sql/db_struct.sql @@ -0,0 +1,1063 @@ +/* +SQLyog Community v13.1.6 (64 bit) +MySQL - 8.0.21 : Database - cvevulner +********************************************************************* +*/ + +/*!40101 SET NAMES utf8 */; + +/*!40101 SET SQL_MODE=''*/; + +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; +CREATE DATABASE /*!32312 IF NOT EXISTS*/`cvevulner` /*!40100 DEFAULT CHARACTER SET utf8 */ /*!80016 DEFAULT ENCRYPTION='N' */; + +USE `cvevulner`; + +/*Table structure for table `cve_admin_user` */ + +DROP TABLE IF EXISTS `cve_admin_user`; + +CREATE TABLE `cve_admin_user` ( + `user_id` int NOT NULL AUTO_INCREMENT, + `user_name` varchar(64) NOT NULL DEFAULT '', + `pass_word` varchar(256) NOT NULL DEFAULT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`user_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_cvrf_sa_record` */ + +DROP TABLE IF EXISTS `cve_cvrf_sa_record`; + +CREATE TABLE `cve_cvrf_sa_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `openeuler_sa_num` varchar(128) NOT NULL DEFAULT '', + `xml_content` text CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, + `cur_md5` varchar(128) NOT NULL DEFAULT '', + `update_md5` varchar(128) NOT NULL DEFAULT '', + `cve_status` tinyint NOT NULL DEFAULT '1', + `is_export` tinyint DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + `pack_name` varchar(512) NOT NULL DEFAULT '', + `cve_num` varchar(1024) NOT NULL DEFAULT '', + `affect_flag` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), + UNIQUE KEY `cve_cvrf_md5` (`cur_md5`), + KEY `cve_cvrf_sa_record_cve_num` (`cve_num`) +) ENGINE=InnoDB AUTO_INCREMENT=378 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_email_list` */ + +DROP TABLE IF EXISTS `cve_email_list`; + +CREATE TABLE `cve_email_list` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `email_name` varchar(512) DEFAULT NULL, + `email_type` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + KEY `cve_email_list_email_type` (`email_type`) +) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_export_record` */ + +DROP TABLE IF EXISTS `cve_export_record`; + +CREATE TABLE `cve_export_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `file_name` varchar(255) NOT NULL DEFAULT '', + `create_time` bigint NOT NULL, + `file_code` varchar(255) NOT NULL DEFAULT '', + `state` tinyint NOT NULL DEFAULT '0', + PRIMARY KEY (`id`), + UNIQUE KEY `file_name` (`file_name`) +) ENGINE=InnoDB AUTO_INCREMENT=194 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_file_hash` */ + +DROP TABLE IF EXISTS `cve_file_hash`; + +CREATE TABLE `cve_file_hash` ( + `id` int NOT NULL AUTO_INCREMENT, + `file_name` varchar(50) DEFAULT NULL, + `file_hash` varchar(50) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=418 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_open_euler` */ + +DROP TABLE IF EXISTS `cve_git_open_euler`; + +CREATE TABLE `cve_git_open_euler` ( + `git_id` bigint NOT NULL AUTO_INCREMENT, + `package_id` bigint NOT NULL DEFAULT '0', + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `origin_url` varchar(512) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `license` varchar(4096) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `feature` varchar(128) NOT NULL DEFAULT '', + `main_tainer` varchar(128) NOT NULL DEFAULT '', + `main_tain_level` tinyint NOT NULL DEFAULT '1', + `release_time` varchar(32) NOT NULL DEFAULT '', + `used_time` varchar(32) NOT NULL DEFAULT '', + `latest_version` varchar(128) NOT NULL DEFAULT '', + `latest_version_time` varchar(32) NOT NULL DEFAULT '', + `issue_count` bigint NOT NULL DEFAULT '0', + `table_id` bigint NOT NULL DEFAULT '0', + `table_name` varchar(128) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '0', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `delete_time` datetime DEFAULT NULL, + `cpe_packname` varchar(512) DEFAULT NULL, + `releases` varchar(128) NOT NULL DEFAULT '', + PRIMARY KEY (`git_id`), + UNIQUE KEY `cve_git_open_euler_package_version` (`package_name`,`version`), + KEY `cve_git_open_euler_package_id` (`package_id`), + KEY `cve_git_open_euler_version` (`version`), + KEY `cve_git_open_euler_table_id` (`table_id`) +) ENGINE=InnoDB AUTO_INCREMENT=7493 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_open_euler_table_relate` */ + +DROP TABLE IF EXISTS `cve_git_open_euler_table_relate`; + +CREATE TABLE `cve_git_open_euler_table_relate` ( + `relate_id` bigint NOT NULL AUTO_INCREMENT, + `git_id` bigint NOT NULL DEFAULT '0', + `brands` varchar(256) NOT NULL DEFAULT '', + `table_name` varchar(256) NOT NULL DEFAULT '', + PRIMARY KEY (`relate_id`), + UNIQUE KEY `cve_git_open_euler_table_gitid_tab_un` (`git_id`,`table_name`), + KEY `cve_git_open_euler_table_relate_git_id` (`git_id`) +) ENGINE=InnoDB AUTO_INCREMENT=6356 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_package_info` */ + +DROP TABLE IF EXISTS `cve_git_package_info`; + +CREATE TABLE `cve_git_package_info` ( + `detail_id` bigint NOT NULL AUTO_INCREMENT, + `git_id` bigint NOT NULL DEFAULT '0', + `git_ids` bigint NOT NULL DEFAULT '0', + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `release` varchar(128) NOT NULL DEFAULT '', + `origin_url` varchar(512) NOT NULL DEFAULT '', + `license` varchar(4096) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `feature` varchar(128) NOT NULL DEFAULT '', + `main_tainer` varchar(128) NOT NULL DEFAULT '', + `main_tain_level` tinyint NOT NULL DEFAULT '1', + `git_url` varchar(512) NOT NULL DEFAULT '', + `summary` varchar(1024) NOT NULL DEFAULT '', + `decription` text CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, + `build_require` varchar(4096) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '0', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `delete_time` datetime DEFAULT NULL, + PRIMARY KEY (`detail_id`), + UNIQUE KEY `cve_git_package_info_package_version` (`package_name`,`version`), + KEY `cve_git_package_info_git_id` (`git_id`), + KEY `cve_git_package_info_git_ids` (`git_ids`), + KEY `cve_git_package_info_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=4890 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_package_table` */ + +DROP TABLE IF EXISTS `cve_git_package_table`; + +CREATE TABLE `cve_git_package_table` ( + `table_id` bigint NOT NULL AUTO_INCREMENT, + `table_name` varchar(128) NOT NULL DEFAULT '', + `brands` varchar(256) NOT NULL DEFAULT '', + PRIMARY KEY (`table_id`) +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_repo_groups` */ + +DROP TABLE IF EXISTS `cve_git_repo_groups`; + +CREATE TABLE `cve_git_repo_groups` ( + `group_id` bigint NOT NULL AUTO_INCREMENT, + `group_name` varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY (`group_id`), + KEY `cve_git_repo_groups_group_name` (`group_name`) +) ENGINE=InnoDB AUTO_INCREMENT=153 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_sub_pack` */ + +DROP TABLE IF EXISTS `cve_git_sub_pack`; + +CREATE TABLE `cve_git_sub_pack` ( + `sub_id` bigint NOT NULL AUTO_INCREMENT, + `detail_id` bigint NOT NULL DEFAULT '0', + `ids` bigint NOT NULL DEFAULT '0', + `sub_pack_name` varchar(256) NOT NULL DEFAULT '', + PRIMARY KEY (`sub_id`), + KEY `cve_git_sub_pack_detail_id` (`detail_id`), + KEY `cve_git_sub_pack_ids` (`ids`) +) ENGINE=InnoDB AUTO_INCREMENT=14236 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_sub_pack_provides` */ + +DROP TABLE IF EXISTS `cve_git_sub_pack_provides`; + +CREATE TABLE `cve_git_sub_pack_provides` ( + `provide_id` bigint NOT NULL AUTO_INCREMENT, + `sub_id` bigint NOT NULL DEFAULT '0', + `ids` bigint NOT NULL DEFAULT '0', + `provide_name` varchar(256) NOT NULL DEFAULT '', + `requiredb` varchar(4096) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + PRIMARY KEY (`provide_id`), + KEY `cve_git_sub_pack_provides_sub_id` (`sub_id`), + KEY `cve_git_sub_pack_provides_ids` (`ids`) +) ENGINE=InnoDB AUTO_INCREMENT=80902 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_sub_pack_require` */ + +DROP TABLE IF EXISTS `cve_git_sub_pack_require`; + +CREATE TABLE `cve_git_sub_pack_require` ( + `require_id` bigint NOT NULL AUTO_INCREMENT, + `sub_id` bigint NOT NULL DEFAULT '0', + `ids` bigint NOT NULL DEFAULT '0', + `require_name` varchar(256) NOT NULL DEFAULT '', + `providedb` varchar(4096) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + PRIMARY KEY (`require_id`), + KEY `cve_git_sub_pack_require_sub_id` (`sub_id`), + KEY `cve_git_sub_pack_require_ids` (`ids`) +) ENGINE=InnoDB AUTO_INCREMENT=106624 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_git_sub_pack_requiredby` */ + +DROP TABLE IF EXISTS `cve_git_sub_pack_requiredby`; + +CREATE TABLE `cve_git_sub_pack_requiredby` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `provide_id` bigint NOT NULL DEFAULT '0', + `requiredb` varchar(256) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + KEY `cve_git_sub_pack_requiredby_provide_id` (`provide_id`) +) ENGINE=InnoDB AUTO_INCREMENT=121916 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_gite_origin_issue` */ + +DROP TABLE IF EXISTS `cve_gite_origin_issue`; + +CREATE TABLE `cve_gite_origin_issue` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `issue_id` bigint NOT NULL DEFAULT '0', + `url` varchar(255) NOT NULL DEFAULT '', + `number` varchar(50) NOT NULL DEFAULT '', + `state` varchar(50) NOT NULL DEFAULT '', + `title` varchar(255) DEFAULT NULL, + `cve_number` varchar(255) NOT NULL DEFAULT '', + `body` longtext, + `issue_create` varchar(255) NOT NULL DEFAULT '', + `issue_assignee` varchar(255) DEFAULT NULL, + `repo_path` varchar(255) NOT NULL DEFAULT '', + `repo_url` varchar(255) NOT NULL DEFAULT '', + `issue_type` varchar(255) NOT NULL DEFAULT '', + `issue_exist_tpl` tinyint(1) NOT NULL DEFAULT '0', + `security_hole` tinyint(1) NOT NULL DEFAULT '0', + `issue_create_at` datetime DEFAULT NULL, + `issue_update_at` datetime DEFAULT NULL, + `issue_finish_at` datetime DEFAULT NULL, + `grab_time` datetime NOT NULL, + `proc_status` tinyint NOT NULL DEFAULT '0', + `inf_product` varchar(255) DEFAULT NULL, + `repo_desc` varchar(2048) DEFAULT NULL, + `issue_state` varchar(50) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `issue_id` (`issue_id`), + UNIQUE KEY `number` (`number`,`cve_number`,`repo_path`) +) ENGINE=InnoDB AUTO_INCREMENT=11177 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_gite_repo` */ + +DROP TABLE IF EXISTS `cve_gite_repo`; + +CREATE TABLE `cve_gite_repo` ( + `repo_id` bigint NOT NULL AUTO_INCREMENT, + `group_id` bigint NOT NULL DEFAULT '0', + `org_id` bigint NOT NULL DEFAULT '0', + `org_path` varchar(128) DEFAULT NULL, + `path` varchar(512) DEFAULT NULL, + `status` tinyint DEFAULT '0', + `public` tinyint(1) NOT NULL DEFAULT '0', + `private` tinyint(1) NOT NULL DEFAULT '0', + `fork` tinyint(1) NOT NULL DEFAULT '0', + `create_time` varchar(255) DEFAULT NULL, + `update_time` varchar(255) DEFAULT NULL, + `delete_time` varchar(255) DEFAULT NULL, + PRIMARY KEY (`repo_id`), + KEY `cve_gite_repo_group_id` (`group_id`) +) ENGINE=InnoDB AUTO_INCREMENT=263949 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_gite_repo_branch` */ + +DROP TABLE IF EXISTS `cve_gite_repo_branch`; + +CREATE TABLE `cve_gite_repo_branch` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `repo_id` bigint NOT NULL DEFAULT '0', + `protection_url` varchar(255) NOT NULL DEFAULT '', + `name` varchar(255) NOT NULL DEFAULT '', + `protected` varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + KEY `cve_gite_repo_branch_repo_id` (`repo_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_gite_repo_member` */ + +DROP TABLE IF EXISTS `cve_gite_repo_member`; + +CREATE TABLE `cve_gite_repo_member` ( + `repo_id` bigint NOT NULL AUTO_INCREMENT, + `group_id` bigint NOT NULL DEFAULT '0', + `member_name` varchar(512) DEFAULT NULL, + `member_type` varchar(128) DEFAULT NULL, + `create_time` varchar(255) DEFAULT NULL, + `update_time` varchar(255) DEFAULT NULL, + `delete_time` varchar(255) DEFAULT NULL, + PRIMARY KEY (`repo_id`), + KEY `cve_gite_repo_member_group_id` (`group_id`), + CONSTRAINT `member_groups_group_id` FOREIGN KEY (`group_id`) REFERENCES `cve_git_repo_groups` (`group_id`) +) ENGINE=InnoDB AUTO_INCREMENT=16553 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_ip_white` */ + +DROP TABLE IF EXISTS `cve_ip_white`; + +CREATE TABLE `cve_ip_white` ( + `ip_id` int NOT NULL AUTO_INCREMENT, + `machine_name` varchar(128) NOT NULL DEFAULT '', + `machine_ip` varchar(128) NOT NULL DEFAULT '', + `access_count` bigint NOT NULL DEFAULT '0', + PRIMARY KEY (`ip_id`), + KEY `cve_ip_white_machine_ip` (`machine_ip`) +) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_assignee` */ + +DROP TABLE IF EXISTS `cve_issue_assignee`; + +CREATE TABLE `cve_issue_assignee` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `assignee` varchar(255) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `assignee` (`assignee`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_create_record` */ + +DROP TABLE IF EXISTS `cve_issue_create_record`; + +CREATE TABLE `cve_issue_create_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `cve_desc` varchar(8192) NOT NULL DEFAULT '', + `cve_level` varchar(32) NOT NULL DEFAULT '', + `cve_version` varchar(128) NOT NULL DEFAULT '', + `repair_time` varchar(32) NOT NULL DEFAULT '', + `pack_name` varchar(512) NOT NULL DEFAULT '', + `nvd_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `n_vector_value` varchar(256) NOT NULL DEFAULT '', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + `status` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + KEY `cve_issue_create_record_cve_id` (`cve_id`), + KEY `cve_issue_create_record_cve_num` (`cve_num`) +) ENGINE=InnoDB AUTO_INCREMENT=1517 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_hooks` */ + +DROP TABLE IF EXISTS `cve_issue_hooks`; + +CREATE TABLE `cve_issue_hooks` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `issue_id` bigint NOT NULL DEFAULT '0', + `issue_num` varchar(64) NOT NULL DEFAULT '', + `hook_id` bigint NOT NULL DEFAULT '0', + `owner` varchar(128) NOT NULL DEFAULT '', + `repo` varchar(128) NOT NULL DEFAULT '', + `hook_url` varchar(512) NOT NULL DEFAULT '', + `push_event` tinyint NOT NULL DEFAULT '1', + `tag_push_events` tinyint NOT NULL DEFAULT '1', + `issues_events` tinyint NOT NULL DEFAULT '1', + `note_events` tinyint NOT NULL DEFAULT '1', + `merge_requests_events` tinyint NOT NULL DEFAULT '1', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `cve_issue_hooks_owner_repo_status_un` (`owner`,`repo`,`hook_url`,`status`), + KEY `cve_issue_hooks_Cve_id` (`cve_id`), + KEY `cve_issue_hooks_issue_num` (`issue_num`) +) ENGINE=InnoDB AUTO_INCREMENT=37 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_repo_whitelist` */ + +DROP TABLE IF EXISTS `cve_issue_repo_whitelist`; + +CREATE TABLE `cve_issue_repo_whitelist` ( + `repo_id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `branchs` varchar(512) DEFAULT NULL, + `status` tinyint NOT NULL DEFAULT '0', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`repo_id`), + KEY `cve_issue_repo_whitelist_package_name` (`package_name`), + KEY `cve_issue_repo_whitelist_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=338 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_statistics_mail_list` */ + +DROP TABLE IF EXISTS `cve_issue_statistics_mail_list`; + +CREATE TABLE `cve_issue_statistics_mail_list` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `email_name` varchar(256) NOT NULL DEFAULT '', + `email_type` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `email_name` (`email_name`) +) ENGINE=InnoDB AUTO_INCREMENT=106 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_template` */ + +DROP TABLE IF EXISTS `cve_issue_template`; + +CREATE TABLE `cve_issue_template` ( + `template_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `owned_component` varchar(256) NOT NULL DEFAULT '', + `owned_version` varchar(256) NOT NULL DEFAULT '', + `nvd_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `openeuler_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `nvd_vector` varchar(256) NOT NULL DEFAULT '', + `openeuler_vector` varchar(256) NOT NULL DEFAULT '', + `cve_brief` varchar(4096) NOT NULL DEFAULT '', + `cve_analysis` varchar(4096) NOT NULL DEFAULT '', + `principle_analysis` varchar(4096) NOT NULL DEFAULT '', + `affected_version` varchar(256) NOT NULL DEFAULT '', + `solution` varchar(1024) NOT NULL DEFAULT '', + `issue_id` bigint NOT NULL DEFAULT '0', + `issue_num` varchar(64) NOT NULL DEFAULT '', + `issue_assignee` varchar(128) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '0', + `status_name` varchar(128) DEFAULT NULL, + `issue_status` tinyint NOT NULL DEFAULT '0', + `issue_label` varchar(256) NOT NULL DEFAULT '', + `owner` varchar(128) NOT NULL DEFAULT '', + `repo` varchar(128) NOT NULL DEFAULT '', + `title` varchar(512) NOT NULL DEFAULT '', + `issue_type` varchar(64) NOT NULL DEFAULT '', + `collaborators` varchar(128) DEFAULT NULL, + `milestone` varchar(64) DEFAULT NULL, + `program` varchar(64) DEFAULT NULL, + `security_hole` tinyint NOT NULL DEFAULT '0', + `cve_level` varchar(32) DEFAULT NULL, + `comment_id` bigint DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `delete_time` datetime NOT NULL, + `error_description` text CHARACTER SET utf8 COLLATE utf8_general_ci, + `mt_audit_flag` tinyint NOT NULL DEFAULT '0', + `sa_audit_flag` tinyint NOT NULL DEFAULT '0', + `op_audit_flag` tinyint NOT NULL DEFAULT '0', + PRIMARY KEY (`template_id`), + UNIQUE KEY `cve_issue_template_cvn_issnu` (`cve_num`,`issue_num`), + UNIQUE KEY `cve_issue_template_num_compone_versio` (`cve_num`,`owned_component`,`owned_version`), + KEY `cve_issue_template_cve_id` (`cve_id`), + KEY `cve_issue_template_issue_num` (`issue_num`) +) ENGINE=InnoDB AUTO_INCREMENT=2663 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_open_euler_s_a` */ + +DROP TABLE IF EXISTS `cve_open_euler_s_a`; + +CREATE TABLE `cve_open_euler_s_a` ( + `openeuler_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `public_date` varchar(16) DEFAULT NULL, + `openeuler_sa_num` varchar(128) NOT NULL DEFAULT '', + PRIMARY KEY (`openeuler_id`), + UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), + KEY `cve_open_euler_s_a_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=2925 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_open_guss_yaml` */ + +DROP TABLE IF EXISTS `cve_open_guss_yaml`; + +CREATE TABLE `cve_open_guss_yaml` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `origin_url` varchar(512) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + `cpe_name` varchar(1024) NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `package_name_version` (`package_name`,`version`), + KEY `cve_open_guss_yaml_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=144 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_excel` */ + +DROP TABLE IF EXISTS `cve_origin_excel`; + +CREATE TABLE `cve_origin_excel` ( + `cve_id` bigint NOT NULL AUTO_INCREMENT, + `cve_num` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `cve_url` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `cve_version` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `pack_name` varchar(256) DEFAULT NULL, + `score_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `nvd_score` decimal(10,1) DEFAULT NULL, + `cve_level` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `cve_desc` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci, + `repair_time` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `vector_value` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `attack_vector` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `access_vector` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `attack_complexity` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `access_complexity` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `privilege_required` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `user_interaction` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `scope` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `confidentiality` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `integrity` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `availability` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `authentication` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `cve_status` tinyint DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `delete_time` datetime DEFAULT NULL, + `is_export` tinyint DEFAULT '1', + PRIMARY KEY (`cve_id`) USING BTREE, + UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`,`cve_version`,`pack_name`) +) ENGINE=InnoDB AUTO_INCREMENT=443 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; + +/*Table structure for table `cve_origin_upstream` */ + +DROP TABLE IF EXISTS `cve_origin_upstream`; + +CREATE TABLE `cve_origin_upstream` ( + `cve_id` bigint NOT NULL AUTO_INCREMENT, + `cve_un_ids` varchar(256) NOT NULL DEFAULT '', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `update_type` varchar(32) NOT NULL DEFAULT '', + `cve_packname` varchar(512) DEFAULT NULL, + `git_packname` varchar(512) DEFAULT NULL, + `cve_title` varchar(1024) DEFAULT NULL, + `affect_porduct` varchar(512) DEFAULT NULL, + `cnnvd_id` varchar(256) DEFAULT NULL, + `cnvd_id` varchar(256) DEFAULT NULL, + `published_date` varchar(32) DEFAULT NULL, + `vul_status` varchar(64) DEFAULT NULL, + `cve_status` tinyint NOT NULL DEFAULT '0', + `version` varchar(64) NOT NULL, + `affected_scope` varchar(512) DEFAULT NULL, + `attack_link` varchar(512) DEFAULT NULL, + `is_exit` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`cve_id`), + UNIQUE KEY `cve_un_ids` (`cve_un_ids`), + KEY `cve_origin_upstream_cve_num` (`cve_num`), + KEY `cve_origin_upstream_cve_packname` (`cve_packname`), + KEY `cve_origin_upstream_git_packname` (`git_packname`), + KEY `cve_origin_upstream_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=2148 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_config` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_config`; + +CREATE TABLE `cve_origin_upstream_config` ( + `conf_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `nodes` varchar(32) DEFAULT NULL, + PRIMARY KEY (`conf_id`), + KEY `cve_origin_upstream_config_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_config_node` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_config_node`; + +CREATE TABLE `cve_origin_upstream_config_node` ( + `node_id` bigint NOT NULL AUTO_INCREMENT, + `conf_id` bigint NOT NULL DEFAULT '0', + `operator` varchar(256) DEFAULT NULL, + PRIMARY KEY (`node_id`), + KEY `cve_origin_upstream_config_node_conf_id` (`conf_id`) +) ENGINE=InnoDB AUTO_INCREMENT=26544 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_config_node_cpe` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_config_node_cpe`; + +CREATE TABLE `cve_origin_upstream_config_node_cpe` ( + `cpe_id` bigint NOT NULL AUTO_INCREMENT, + `node_id` bigint NOT NULL DEFAULT '0', + `cpe_uri` varchar(1024) DEFAULT NULL, + `cpe_match` varchar(1024) DEFAULT NULL, + `vulner_able` varchar(64) DEFAULT NULL, + PRIMARY KEY (`cpe_id`), + KEY `cve_origin_upstream_config_node_cpe_node_id` (`node_id`) +) ENGINE=InnoDB AUTO_INCREMENT=773120 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_desc` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_desc`; + +CREATE TABLE `cve_origin_upstream_desc` ( + `desc_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `en_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, + `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, + PRIMARY KEY (`desc_id`), + KEY `cve_origin_upstream_desc_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_event` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_event`; + +CREATE TABLE `cve_origin_upstream_event` ( + `event_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `title` varchar(512) DEFAULT NULL, + `date` varchar(32) DEFAULT NULL, + `url` varchar(512) DEFAULT NULL, + `description` text CHARACTER SET utf8 COLLATE utf8_general_ci, + PRIMARY KEY (`event_id`), + KEY `cve_origin_upstream_event_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_fix_suggest` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_fix_suggest`; + +CREATE TABLE `cve_origin_upstream_fix_suggest` ( + `fix_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `detail` varchar(1024) DEFAULT NULL, + PRIMARY KEY (`fix_id`), + KEY `cve_origin_upstream_fix_suggest_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14032 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_fix_suggest_ref` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_fix_suggest_ref`; + +CREATE TABLE `cve_origin_upstream_fix_suggest_ref` ( + `fix_ref_id` bigint NOT NULL AUTO_INCREMENT, + `fix_id` bigint NOT NULL DEFAULT '0', + `refsource` varchar(512) DEFAULT NULL, + `url` varchar(1024) DEFAULT NULL, + `name` varchar(1024) DEFAULT NULL, + PRIMARY KEY (`fix_ref_id`), + KEY `cve_origin_upstream_fix_suggest_ref_fix_id` (`fix_id`) +) ENGINE=InnoDB AUTO_INCREMENT=44 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_fix_suggest_ref_tag` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_fix_suggest_ref_tag`; + +CREATE TABLE `cve_origin_upstream_fix_suggest_ref_tag` ( + `tag_id` bigint NOT NULL AUTO_INCREMENT, + `fix_ref_id` bigint NOT NULL DEFAULT '0', + `name` varchar(512) DEFAULT NULL, + PRIMARY KEY (`tag_id`), + KEY `cve_origin_upstream_fix_suggest_ref_tag_fix_ref_id` (`fix_ref_id`) +) ENGINE=InnoDB AUTO_INCREMENT=130 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_impact` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_impact`; + +CREATE TABLE `cve_origin_upstream_impact` ( + `impact_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `impact` varchar(32) DEFAULT NULL, + PRIMARY KEY (`impact_id`), + KEY `cve_origin_upstream_impact_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_impact_score` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_impact_score`; + +CREATE TABLE `cve_origin_upstream_impact_score` ( + `score_id` bigint NOT NULL AUTO_INCREMENT, + `impact_id` bigint NOT NULL DEFAULT '0', + `base_met_v3` tinyint DEFAULT NULL, + `base_met_v2` tinyint DEFAULT NULL, + `cvss_v3` tinyint DEFAULT NULL, + `cvss_v2` tinyint DEFAULT NULL, + `score_status` tinyint DEFAULT '1', + PRIMARY KEY (`score_id`), + KEY `cve_origin_upstream_impact_score_impact_id` (`impact_id`) +) ENGINE=InnoDB AUTO_INCREMENT=28169 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_impact_score_v2` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_impact_score_v2`; + +CREATE TABLE `cve_origin_upstream_impact_score_v2` ( + `v2_id` bigint NOT NULL AUTO_INCREMENT, + `score_id` bigint NOT NULL DEFAULT '0', + `acinsuf_info` varchar(64) DEFAULT NULL, + `base_score` decimal(10,1) DEFAULT NULL, + `vector_value` varchar(256) DEFAULT NULL, + `access_complexity` varchar(64) DEFAULT NULL, + `authentication` varchar(64) DEFAULT NULL, + `availability_impact` varchar(64) DEFAULT NULL, + `version` varchar(64) DEFAULT NULL, + `confidentiality_impact` varchar(64) DEFAULT NULL, + `integrity_impact` varchar(64) DEFAULT NULL, + `access_vector` varchar(64) DEFAULT NULL, + `user_interaction_required` varchar(64) DEFAULT NULL, + `severity` varchar(64) DEFAULT NULL, + `obtain_user_privilege` varchar(64) DEFAULT NULL, + `obtain_all_privilege` varchar(64) DEFAULT NULL, + `obtain_other_privilege` varchar(64) DEFAULT NULL, + `impact_score` decimal(10,1) DEFAULT NULL, + `exploitability_score` decimal(10,1) DEFAULT NULL, + `cve_level` varchar(32) DEFAULT NULL, + PRIMARY KEY (`v2_id`), + KEY `cve_origin_upstream_impact_score_v2_score_id` (`score_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_impact_score_v3` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_impact_score_v3`; + +CREATE TABLE `cve_origin_upstream_impact_score_v3` ( + `v3_id` bigint NOT NULL AUTO_INCREMENT, + `score_id` bigint NOT NULL DEFAULT '0', + `base_score` decimal(10,1) DEFAULT NULL, + `vector_value` varchar(256) DEFAULT NULL, + `attack_complexity` varchar(64) DEFAULT NULL, + `attack_vector` varchar(64) DEFAULT NULL, + `availability_impact` varchar(64) DEFAULT NULL, + `base_severity` varchar(64) DEFAULT NULL, + `user_interaction` varchar(64) DEFAULT NULL, + `privilege_required` varchar(64) DEFAULT NULL, + `version` varchar(64) DEFAULT NULL, + `confidentiality_impact` varchar(64) DEFAULT NULL, + `integrity_impact` varchar(64) DEFAULT NULL, + `scope` varchar(64) DEFAULT NULL, + `impact_score` decimal(10,1) DEFAULT NULL, + `exploitability_score` decimal(10,1) DEFAULT NULL, + `cve_level` varchar(32) DEFAULT NULL, + PRIMARY KEY (`v3_id`), + KEY `cve_origin_upstream_impact_score_v3_score_id` (`score_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_poc` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_poc`; + +CREATE TABLE `cve_origin_upstream_poc` ( + `poc_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `source` varchar(512) DEFAULT NULL, + `date` varchar(32) DEFAULT NULL, + `path` varchar(512) DEFAULT NULL, + `dbindex` varchar(512) DEFAULT NULL, + `url` varchar(512) DEFAULT NULL, + `desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, + PRIMARY KEY (`poc_id`), + KEY `cve_origin_upstream_poc_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_reference` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_reference`; + +CREATE TABLE `cve_origin_upstream_reference` ( + `ref_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `name` varchar(512) DEFAULT NULL, + `refsource` varchar(512) DEFAULT NULL, + `url` varchar(512) DEFAULT NULL, + `tags` text CHARACTER SET utf8 COLLATE utf8_general_ci, + PRIMARY KEY (`ref_id`), + KEY `cve_origin_upstream_reference_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=271924 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_origin_upstream_vul_type` */ + +DROP TABLE IF EXISTS `cve_origin_upstream_vul_type`; + +CREATE TABLE `cve_origin_upstream_vul_type` ( + `vul_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cwe` varchar(256) DEFAULT NULL, + `en_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, + `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, + PRIMARY KEY (`vul_id`), + KEY `cve_origin_upstream_vul_type_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=14032 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_other_user` */ + +DROP TABLE IF EXISTS `cve_other_user`; + +CREATE TABLE `cve_other_user` ( + `user_id` int NOT NULL AUTO_INCREMENT, + `user_name` varchar(64) NOT NULL DEFAULT '', + `pass_word` varchar(256) NOT NULL DEFAULT '', + `aes_key` varchar(512) NOT NULL DEFAULT '', + `expiration_time` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `delete_time` datetime DEFAULT NULL, + PRIMARY KEY (`user_id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_package` */ + +DROP TABLE IF EXISTS `cve_package`; + +CREATE TABLE `cve_package` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `sec_id` bigint NOT NULL DEFAULT '0', + `pack_name` varchar(1024) DEFAULT NULL, + `pack_url` varchar(2048) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `cve_package_sec_id` (`sec_id`) +) ENGINE=InnoDB AUTO_INCREMENT=82349 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_package_cpe` */ + +DROP TABLE IF EXISTS `cve_package_cpe`; + +CREATE TABLE `cve_package_cpe` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `packname` varchar(512) DEFAULT NULL, + `cpe_packname` varchar(512) DEFAULT NULL, + `create_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `cve_package_cpe_packname` (`packname`) +) ENGINE=InnoDB AUTO_INCREMENT=2480 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_sa_file_list` */ + +DROP TABLE IF EXISTS `cve_sa_file_list`; + +CREATE TABLE `cve_sa_file_list` ( + `file_id` bigint NOT NULL AUTO_INCREMENT, + `file_name` varchar(512) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`file_id`) +) ENGINE=InnoDB AUTO_INCREMENT=67 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_sa_number` */ + +DROP TABLE IF EXISTS `cve_sa_number`; + +CREATE TABLE `cve_sa_number` ( + `sa_id` bigint NOT NULL AUTO_INCREMENT, + `openeuler_sa_num` varchar(128) NOT NULL DEFAULT '', + `sa_years` varchar(16) NOT NULL DEFAULT '', + `sa_num` bigint NOT NULL DEFAULT '0', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`sa_id`) +) ENGINE=InnoDB AUTO_INCREMENT=37984 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_score` */ + +DROP TABLE IF EXISTS `cve_score`; + +CREATE TABLE `cve_score` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `nvd_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `openeuler_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `n_vector_value` varchar(256) NOT NULL DEFAULT '', + `o_vector_value` varchar(256) NOT NULL DEFAULT '', + `n_attack_vector` varchar(64) NOT NULL DEFAULT '', + `o_attack_vector` varchar(64) NOT NULL DEFAULT '', + `n_attack_complexity` varchar(64) NOT NULL DEFAULT '', + `o_attack_complexity` varchar(64) NOT NULL DEFAULT '', + `n_privilege_required` varchar(64) NOT NULL DEFAULT '', + `o_privilege_required` varchar(64) NOT NULL DEFAULT '', + `n_user_interaction` varchar(64) NOT NULL DEFAULT '', + `o_user_interaction` varchar(64) NOT NULL DEFAULT '', + `n_scope` varchar(64) NOT NULL DEFAULT '', + `o_scope` varchar(64) NOT NULL DEFAULT '', + `n_confidentiality` varchar(64) NOT NULL DEFAULT '', + `o_confidentiality` varchar(64) NOT NULL DEFAULT '', + `n_integrity` varchar(64) NOT NULL DEFAULT '', + `o_integrity` varchar(64) NOT NULL DEFAULT '', + `n_availability` varchar(64) NOT NULL DEFAULT '', + `o_availability` varchar(64) NOT NULL DEFAULT '', + `score_type` varchar(16) NOT NULL DEFAULT '', + `openeuler_id` bigint DEFAULT NULL, + `n_score_status` tinyint NOT NULL DEFAULT '0', + `o_score_status` tinyint NOT NULL DEFAULT '0', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `delete_time` datetime NOT NULL, + `n_access_vector` varchar(256) DEFAULT NULL, + `o_access_vector` varchar(256) DEFAULT NULL, + `n_access_complexity` varchar(64) DEFAULT NULL, + `o_access_complexity` varchar(64) DEFAULT NULL, + `n_authentication` varchar(64) DEFAULT NULL, + `o_authentication` varchar(64) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `openeuler_id` (`openeuler_id`), + KEY `cve_score_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=2925 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_score_record` */ + +DROP TABLE IF EXISTS `cve_score_record`; + +CREATE TABLE `cve_score_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `nvd_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `n_vector_value` varchar(256) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '0', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `cve_score_record_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=3725 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_security_notice` */ + +DROP TABLE IF EXISTS `cve_security_notice`; + +CREATE TABLE `cve_security_notice` ( + `sec_id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `openeuler_id` bigint DEFAULT NULL, + `introduction` varchar(256) DEFAULT NULL, + `summary` varchar(256) DEFAULT NULL, + `theme` varchar(2048) DEFAULT NULL, + `description` text CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, + `influence_component` varchar(256) DEFAULT NULL, + `affect_product` varchar(256) DEFAULT NULL, + `reference_link` varchar(1024) DEFAULT NULL, + `sec_status` tinyint NOT NULL DEFAULT '0', + `affect_status` varchar(16) DEFAULT NULL, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `delete_time` datetime NOT NULL, + PRIMARY KEY (`sec_id`), + UNIQUE KEY `openeuler_id` (`openeuler_id`), + KEY `cve_security_notice_cve_id` (`cve_id`) +) ENGINE=InnoDB AUTO_INCREMENT=2937 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_security_reviewer` */ + +DROP TABLE IF EXISTS `cve_security_reviewer`; + +CREATE TABLE `cve_security_reviewer` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name_space` varchar(255) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '0', + PRIMARY KEY (`id`), + UNIQUE KEY `name_space` (`name_space`) +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_spec_error` */ + +DROP TABLE IF EXISTS `cve_spec_error`; + +CREATE TABLE `cve_spec_error` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_num` varchar(256) NOT NULL DEFAULT '', + `cve_desc` varchar(8192) NOT NULL DEFAULT '', + `cve_owner` varchar(256) NOT NULL DEFAULT '', + `cve_status` tinyint NOT NULL DEFAULT '1', + `pack_name` varchar(512) NOT NULL DEFAULT '', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `cve_spec_error_cve_num` (`cve_num`), + KEY `cve_spec_error_cve_owner` (`cve_owner`) +) ENGINE=InnoDB AUTO_INCREMENT=20023 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_spec_issue_assigness` */ + +DROP TABLE IF EXISTS `cve_spec_issue_assigness`; + +CREATE TABLE `cve_spec_issue_assigness` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `issue_assignee` varchar(256) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `cve_spec_issue_assigness_package_name` (`package_name`) +) ENGINE=InnoDB AUTO_INCREMENT=1195 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_vuln_center` */ + +DROP TABLE IF EXISTS `cve_vuln_center`; + +CREATE TABLE `cve_vuln_center` ( + `cve_id` bigint NOT NULL AUTO_INCREMENT, + `cve_num` varchar(256) NOT NULL DEFAULT '', + `cve_level` varchar(32) DEFAULT NULL, + `cve_desc` varchar(8192) NOT NULL DEFAULT '', + `cve_status` tinyint NOT NULL DEFAULT '0', + `cve_version` varchar(128) NOT NULL DEFAULT '', + `repair_time` varchar(32) NOT NULL DEFAULT '', + `pack_name` varchar(512) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '', + `cve_url` varchar(2048) NOT NULL DEFAULT '', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `delete_time` datetime NOT NULL, + `is_export` tinyint NOT NULL DEFAULT '0', + `data_source` tinyint NOT NULL DEFAULT '1', + `cve_detail_url` varchar(1024) NOT NULL DEFAULT '', + `organizate_id` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`cve_id`), + UNIQUE KEY `cve_vuln_center_num_pack_v_un` (`cve_num`,`cve_version`,`pack_name`), + KEY `cve_vuln_center_cve_num` (`cve_num`) +) ENGINE=InnoDB AUTO_INCREMENT=2931 DEFAULT CHARSET=utf8; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; diff --git a/cve-vulner-manager/errcode/errcode.go b/cve-vulner-manager/errcode/errcode.go new file mode 100644 index 0000000000000000000000000000000000000000..0fe38830338826f120a6e5576d53716f29e82ad3 --- /dev/null +++ b/cve-vulner-manager/errcode/errcode.go @@ -0,0 +1,52 @@ +package errcode + +const ( + RecodeOk = "200" + RecodeDbErr = "4001" + RecodeNodata = "4002" + RecodeDataExist = "4003" + RecodeDataErr = "4004" + RecodeSessionErr = "4101" + RecodeLoginErr = "4102" + RecodeParamErr = "4103" + RecodeUserErr = "4104" + RecodeRoleErr = "4105" + RecodePwdErr = "4106" + RecodeReqErr = "4201" + RecodeIpErr = "4202" + RecodeThirdErr = "4301" + RecodeIoErr = "4302" + RecodeServerErr = "4500" + RecodeUnknowErr = "4501" + RecodeDataMissErr = "4502" +) + +var recodeText = map[string]string{ + RecodeOk: "成功", + RecodeDbErr: "数据库查询错误", + RecodeNodata: "无数据", + RecodeDataExist: "数据已存在", + RecodeDataErr: "数据错误", + RecodeSessionErr: "用户未登录", + RecodeLoginErr: "用户登录失败", + RecodeParamErr: "请求参数错误", + RecodeUserErr: "用户不存在或未激活", + RecodeRoleErr: "登录身份错误", + RecodePwdErr: "密码错误", + RecodeReqErr: "非法请求或请求次数受限", + RecodeIpErr: "IP受限", + RecodeThirdErr: "第三方系统错误", + RecodeIoErr: "文件读写错误", + RecodeServerErr: "内部错误", + RecodeUnknowErr: "未知错误", + RecodeDataMissErr: "数据存在缺失(请检查cve编号、组件、版本号等)", +} + +//RecodeText get resp string by code +func RecodeText(code string) string { + str, ok := recodeText[code] + if ok { + return str + } + return RecodeText(RecodeUnknowErr) +} diff --git a/cve-vulner-manager/git b/cve-vulner-manager/git new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cve-vulner-manager/go.mod b/cve-vulner-manager/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..16aab381a79633f695a59935852d43cc4333e4a9 --- /dev/null +++ b/cve-vulner-manager/go.mod @@ -0,0 +1,22 @@ +module cvevulner + +go 1.14 + +require ( + github.com/360EntSecGroup-Skylar/excelize/v2 v2.3.0 + github.com/antchfx/htmlquery v1.2.4 + github.com/astaxie/beego v1.12.3 + github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/go-sql-driver/mysql v1.5.0 + github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect + github.com/lib/pq v1.8.0 // indirect + github.com/pkg/errors v0.9.1 + github.com/robfig/cron/v3 v3.0.1 + github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect + github.com/smartystreets/goconvey v1.6.4 + github.com/xuri/excelize/v2 v2.5.0 + golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df + gopkg.in/yaml.v2 v2.3.0 +) diff --git a/cve-vulner-manager/go.sum b/cve-vulner-manager/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..7b21725107b66ae83495a9513e4c551ba87e8edb --- /dev/null +++ b/cve-vulner-manager/go.sum @@ -0,0 +1,246 @@ +github.com/360EntSecGroup-Skylar/excelize/v2 v2.3.0 h1:tDWYNCJrpNnlNg8mVdlzAzPjlPaRbsA/kS8H9LczleQ= +github.com/360EntSecGroup-Skylar/excelize/v2 v2.3.0/go.mod h1:Uwb0d1GgxJieUWZG5WylTrgQ2SrldfjagAxheU8W6MQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/astaxie/beego v1.12.3 h1:SAQkdD2ePye+v8Gn1r4X6IKZM1wd28EyUOVQ3PDSOOQ= +github.com/astaxie/beego v1.12.3/go.mod h1:p3qIm0Ryx7zeBHLljmd7omloyca1s4yu1a8kM1FkpIA= +github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ= +github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= +github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= +github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= +github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= +github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U= +github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/richardlehane/mscfb v1.0.3 h1:rD8TBkYWkObWO0oLDFCbwMeZ4KoalxQy+QgniCj3nKI= +github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo= +github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= +github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/siddontang/goredis v0.0.0-20150324035039-760763f78400/go.mod h1:DDcKzU3qCuvj/tPnimWSsZZzvk9qvkvrIL5naVBPh5s= +github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= +github.com/xuri/efp v0.0.0-20191019043341-b7dc4fe9aa91/go.mod h1:uBiSUepVYMhGTfDeBKKasV4GpgBlzJ46gXUBAqV8qLk= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 h1:EpI0bqf/eX9SdZDwlMmahKM+CDBgNbsXMhsN28XrM8o= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.5.0 h1:nDDVfX0qaDuGjAvb+5zTd0Bxxoqa1Ffv9B4kiE23PTM= +github.com/xuri/excelize/v2 v2.5.0/go.mod h1:rSu0C3papjzxQA3sdK8cU544TebhrPUoTOaGPIh0Q1A= +github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go new file mode 100644 index 0000000000000000000000000000000000000000..eb6d1a599f65294fd53f9fa2b38a155c058451c3 --- /dev/null +++ b/cve-vulner-manager/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "cvevulner/common" + cve_timed_task "cvevulner/cve-timed-task" + "cvevulner/models" + _ "cvevulner/models" + _ "cvevulner/routers" + "cvevulner/task" + "github.com/astaxie/beego" +) + +func init() { + // Initialize global variables + common.InitGlobal() + // Initialization log + common.LogInit() +} + +func main() { + // init db + dbOk := models.Initdb() + if !dbOk { + println("error: Database initialization failed") + return + } + // Initialize a scheduled task + taskOk := task.InitTask() + if !taskOk { + println("error: Timing task initialization failed, the program ends") + task.StopTask() + return + } + + err := cve_timed_task.StartCveTimedTask() + if err != nil { + return + } + // single run + task.StartTask() + defer task.StopTask() + if beego.BConfig.RunMode == "dev" { + beego.BConfig.WebConfig.DirectoryIndex = true + beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" + } + beego.Run() +} diff --git a/cve-vulner-manager/master b/cve-vulner-manager/master new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cve-vulner-manager/models/common.go b/cve-vulner-manager/models/common.go new file mode 100644 index 0000000000000000000000000000000000000000..586628dfad9ade0a68cd03125311f66066f0267d --- /dev/null +++ b/cve-vulner-manager/models/common.go @@ -0,0 +1,116 @@ +package models + +import ( + "fmt" + "github.com/astaxie/beego/logs" + "time" +) + +//Time2Str format time to string +func Time2Str() string { + const shortForm = "2006-01-02 15:04:05" + t := time.Now() + str := t.Format(shortForm) + return str +} + +//CheckToken Check whether the token is legal +func CheckToken(token string) bool { + var ou OtherUser + err := GetUserByToken(token, &ou) + if err != nil { + logs.Error("token value: ", token, ", token, is forged data, does not exist") + return false + } + now := Time2Str() + logs.Info("token: now: ", now, ",expir: ", ou.ExpirationTime) + if now > ou.ExpirationTime { + return false + } + return true +} + +func OpenEulerScoreProc(openEulerScore float64) (CveLevel string) { + if openEulerScore >= 9.0 { + CveLevel = "Critical" + } else if openEulerScore >= 7.0 && openEulerScore <= 8.9 { + CveLevel = "High" + } else if openEulerScore > 4.0 && openEulerScore <= 6.9 { + CveLevel = "Medium" + } else if openEulerScore <= 4.0 { + CveLevel = "Low" + } + return CveLevel +} + +func SearchGaussCondSa(cveLevel, years, releaseFlag int, searchName string) string { + conditStr := "" + if cveLevel > 0 { + conditStr = fmt.Sprintf(`cve_level_value = %d`, cveLevel) + } + if years > 0 { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`sa_years = %d`, years) + } else { + conditStr = fmt.Sprintf(`sa_years = %d`, years) + } + } + if releaseFlag == 2 { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`status = 3`) + } else { + conditStr = fmt.Sprintf(`status = 3`) + } + } else { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`status = 1`) + } else { + conditStr = fmt.Sprintf(`status = 1`) + } + } + if len(searchName) > 1 { + searchName = "%" + searchName + "%" + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`(gauss_sa_num like '%v' or influence_component like '%v')`, + searchName, searchName) + } else { + conditStr = fmt.Sprintf(`gauss_sa_num like '%v' or influence_component like '%v'`, searchName, searchName) + } + } + return conditStr +} + +func SearchGaussCondCve(releaseFlag, isAffectFlag int, searchName string) string { + conditStr := "" + if len(searchName) > 1 { + searchName = "%" + searchName + "%" + conditStr = fmt.Sprintf(`cve_num like '%v'`, searchName) + } + if releaseFlag == 2 { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`status = 3`) + } else { + conditStr = fmt.Sprintf(`status = 3`) + } + } else { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`status = 1`) + } else { + conditStr = fmt.Sprintf(`status = 1`) + } + } + if isAffectFlag == 3 { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`gauss_id = 0`) + } else { + conditStr = fmt.Sprintf(`gauss_id = 0`) + } + } else if isAffectFlag == 2 { + if len(conditStr) > 1 { + conditStr = conditStr + " and " + fmt.Sprintf(`gauss_id > 0`) + } else { + conditStr = fmt.Sprintf(`gauss_id > 0`) + } + } + return conditStr +} diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go new file mode 100644 index 0000000000000000000000000000000000000000..570a544e403dd952e8ee1e0d9f037e7d3c607e2c --- /dev/null +++ b/cve-vulner-manager/models/cve.go @@ -0,0 +1,1288 @@ +package models + +import ( + "cvevulner/common" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "strings" + "time" +) + +type SaNuberCount struct { + Count int64 +} + +func QueryOriginCveExcel(days string, prcnum int) ([]OriginExcel, int64, error) { + o := orm.NewOrm() + var coe []OriginExcel + num, err := o.Raw("select cve_id,cve_num,cve_url, cve_version, pack_name,"+ + "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity,"+ + "access_complexity,privilege_required,user_interaction,scope,confidentiality,integrity,availability,authentication,"+ + "cve_status,create_time from cve_origin_excel where update_time >= ? and cve_status in (?, ?)"+ + "order by cve_id asc limit ?", days, 0, 1, prcnum).QueryRows(&coe) + if err == nil && num > 0 { + logs.Info("QueryOriginCveExcel, cve_origin_excel, search result: ", num) + } else { + logs.Info("QueryOriginCveExcel, There is currently no manually added or updated cve, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return coe, num, err +} + +func UpdateCveIssueStatusById(status int, issueId int64) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, issue_update_at = ?"+ + " WHERE id = ?", status, common.GetCurTime(), issueId).QueryRow() + return +} + +func LockUpdateCveIssueStatus(status int, issueId int64) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, "+ + "issue_update_at = ? where id = ? and proc_status = 0", + status, common.GetCurTime(), issueId).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("LockUpdateCveIssueStatus, update cve_gite_origin_issue row affected nums: ", num, + ",issueId: ", issueId) + return true + } + return false + } else { + logs.Error("LockUpdateCveIssueStatus, update failed, cve_gite_origin_issue, issueId: ", + issueId, ", err: ", err) + return false + } +} + +func UnLockUpdateCveIssueStatus(beforeStatus, afterStatus int8, updateTime, beforeTime string) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, "+ + "issue_update_at = ? where proc_status = ? and issue_update_at < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateCveIssueStatus, update cve_gite_origin_issue row affected nums: ", num) + return true + } + return false + } else { + logs.Error("UnLockUpdateCveIssueStatus, update failed, cve_gite_origin_issue, err: ", err) + return false + } +} + +func QueryIssueTemplateAll(prcnum int, tmplateId int64, beforeTime string) (ite []IssueTemplate, num int64, err error) { + o := orm.NewOrm() + num, err = o.Raw("select *"+ + " from cve_issue_template where template_id > ? and create_time > ? "+ + "order by template_id asc limit ?", tmplateId, beforeTime, prcnum).QueryRows(&ite) + if err == nil && num > 0 { + logs.Info("QueryIssueTemplateAll, cve_issue_template, search result: ", num) + } else { + logs.Error("QueryIssueTemplateAll, There are currently no new or updated issues, cve_issue_template, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return +} + +func QueryOriginCveIssue(prcnum int) ([]GiteOriginIssue, int64, error) { + o := orm.NewOrm() + var gs []GiteOriginIssue + num, err := o.Raw("select *"+ + " from cve_gite_origin_issue where proc_status = ? "+ + "order by id asc limit ?", 0, prcnum).QueryRows(&gs) + if err == nil && num > 0 { + logs.Info("QueryOriginCveIssue, cve_gite_origin_issue, search result: ", num) + } else { + logs.Error("QueryOriginCveIssue, There are currently no new or updated issues, cve_gite_origin_issue, cur_time:", + common.GetCurTime(), ",err: ", err, gs) + } + return gs, num, err +} + +func QueryOriginCve(days string, prcnum, credibilityLevel int) ([]OriginUpstream, int64, error) { + o := orm.NewOrm() + var gs []OriginUpstream + num, err := o.Raw("select * from cve_origin_upstream where cve_status in (?, ?) "+ + "and credibility_level <= ? "+ + "order by cve_id asc limit ?", 0, 1, credibilityLevel, prcnum).QueryRows(&gs) + if err == nil && num > 0 { + logs.Info("QueryOriginCveIssue, cve_origin_upstream, search result: ", num) + } else { + logs.Info("QueryOriginCveIssue, There are currently no new or updated automated submission cve, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return gs, num, err +} + +func QueryCveDesc(cveId int64) (OriginUpstreamDesc, bool) { + o := orm.NewOrm() + var cveDesc OriginUpstreamDesc + err := o.Raw("select * from cve_origin_upstream_desc where cve_id = ?", cveId).QueryRow(&cveDesc) + if err != nil { + logs.Error(cveId, "QueryCveDesc, "+ + "The original cve description cannot be queried, cveId: ", cveId, ", err: ", err) + return cveDesc, false + } else { + return cveDesc, true + } +} + +func QueryCveImpact(cveId int64) (OriginUpstreamImpact, bool) { + o := orm.NewOrm() + var cveImpact OriginUpstreamImpact + err := o.Raw("select * from cve_origin_upstream_impact where cve_id = ?", cveId).QueryRow(&cveImpact) + if err != nil { + logs.Error(cveId, "QueryCveImpact, cve_origin_upstream_impact, cveId: ", cveId, ",err: ", err) + return cveImpact, false + } else { + return cveImpact, true + } +} + +func QueryCveScore(impactId int64, typex string) (OriginUpstreamImpactScore, bool) { + o := orm.NewOrm() + var cveScore OriginUpstreamImpactScore + if typex == "v3" { + err := o.Raw("select * from cve_origin_upstream_impact_score where "+ + "impact_id = ? and base_met_v3=? and cvss_v3=?", impactId, 1, 1).QueryRow(&cveScore) + if err != nil { + logs.Error(impactId, "QueryCveScore, cve_origin_upstream_impact_score, impactId: ", impactId, ",err: ", err) + return cveScore, false + } else { + return cveScore, true + } + } else { + err := o.Raw("select * from cve_origin_upstream_impact_score where "+ + "impact_id = ? and base_met_v2=? and cvss_v2=?", impactId, 1, 1).QueryRow(&cveScore) + if err != nil { + logs.Error(impactId, "QueryCveScore, cve_origin_upstream_impact_score, impactId: ", impactId, ", err: ", err) + return cveScore, false + } else { + return cveScore, true + } + } + +} + +func QueryCveCvssV3(scoreId int64) (OriginUpstreamImpactScoreV3, bool) { + o := orm.NewOrm() + var cveScoreV3 OriginUpstreamImpactScoreV3 + err := o.Raw("select * from cve_origin_upstream_impact_score_v3 where score_id = ?", scoreId).QueryRow(&cveScoreV3) + if err != nil { + logs.Error("QueryCveCvssV3, cve_origin_upstream_impact_score_v3, scoreId: ", scoreId, ", err: ", err) + return cveScoreV3, false + } else { + return cveScoreV3, true + } +} + +func QueryCveCvssV2(scoreId int64) (OriginUpstreamImpactScoreV2, bool) { + o := orm.NewOrm() + var cveScoreV2 OriginUpstreamImpactScoreV2 + err := o.Raw("select * from cve_origin_upstream_impact_score_v2 where score_id = ?", scoreId).QueryRow(&cveScoreV2) + if err != nil { + logs.Error("QueryCveCvssV2, cve_origin_upstream_impact_score_v2, scoreId: ", scoreId, ", err: ", err) + return cveScoreV2, false + } else { + return cveScoreV2, true + } +} + +func QueryCveByPackName(cveNum, packName string, organizateId int8) (vc []VulnCenter) { + o := orm.NewOrm() + num, err := o.Raw("select * from cve_vuln_center where cve_num = ? and "+ + "pack_name = ? and organizate_id = ? order by cve_id desc", + cveNum, packName, organizateId).QueryRows(&vc) + if err != nil { + logs.Error("QueryCveByPackName, cve_vuln_center, num: ", num, ", err: ", err) + } + return +} + +func QueryCveByNum(cveNum, packName string, organizateId int8) (VulnCenter, bool) { + o := orm.NewOrm() + var cve VulnCenter + err := o.Raw("select * from cve_vuln_center where cve_num = ? and "+ + "pack_name = ? and organizate_id = ? order by cve_id desc limit 1", + cveNum, packName, organizateId).QueryRow(&cve) + if err != nil || cve.CveId == 0 { + logs.Error("QueryCveByNum, cve_vuln_center, cveNum: ", + cveNum, ",packName: ", packName, ", err: ", err) + return cve, false + } else { + return cve, true + } +} + +func UpdateCveStatusExportByNum(updatetime, cveNum string, isExport int, repoPath, version string) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_vuln_center SET "+ + "is_export = ?, update_time = ? where cve_num = ? and pack_name = ? and cve_version = ?", + isExport, updatetime, cveNum, repoPath, version).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("cve_vuln_center row affected nums: ", num, + ",cveNum: ", cveNum, ",updatetime: ", updatetime) + return true + } + return false + } else { + logs.Error("UpdateCveStatusExportByNum, cve_vuln_center, Update failed, cveNum: ", cveNum, + ", updatetime: ", updatetime, ", err: ", err) + return false + } +} + +func QueryScoreByCveId(CveId int64) (Score, bool) { + o := orm.NewOrm() + var score Score + err := o.Raw("select * from cve_score where cve_id = ?", CveId).QueryRow(&score) + if err != nil { + logs.Error(CveId, "QueryScoreByCveId, cve_id: ", CveId, ",err: ", err) + return score, false + } else { + return score, true + } +} + +func QueryOpenEulerSAByCveId(CveId int64) (OpenEulerSA, bool) { + o := orm.NewOrm() + op := OpenEulerSA{CveId: CveId} + err := o.Read(&op, "CveId") + if err == orm.ErrNoRows { + logs.Error("QueryOpenEulerSAByCveId, cveId: ", CveId, ", err: ", err) + return op, false + } else if err == orm.ErrMissPK { + logs.Error("QueryOpenEulerSAByCveId, cveId: ", CveId, ", err: ", err) + return op, false + } else { + return op, true + } +} + +func QuerySecNoticeByCveId(CveId int64) (SecurityNotice, bool) { + o := orm.NewOrm() + var secNotice SecurityNotice + err := o.Raw("select * from cve_security_notice where cve_id = ?", CveId).QueryRow(&secNotice) + if err != nil { + logs.Error("QuerySecNoticeByCveId, cveId: ", CveId, ", err: ", err) + return secNotice, false + } else { + return secNotice, true + } +} + +func CreateSecurityNotice(sec *SecurityNotice) (SecId int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + var num int64 + if num, err = o.Insert(sec); err == nil { + logs.Info("CreateSecurityNotice, insert cve_security_notice success, num: ", num, ",cveNum: ", sec.CveNum) + } else { + logs.Error("CreateSecurityNotice, insert cve_security_notice failed, cveNum:", sec.CveNum, ", err: ", err) + err = o.Rollback() + return 0, err + } + SecId = sec.SecId + _ = o.Commit() + } else { + logs.Error("CreateSecurityNotice, Transaction creation failed, cveNum: ", sec.CveNum, ", errs: ", errs) + return 0, errs + } + return SecId, nil +} + +func CreateScore(score *Score) (Id int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + var num int64 + if num, err = o.Insert(score); err == nil { + logs.Info("CreateScore, insert cve_score success, num: ", num, ",cveNum: ", score.CveNum) + } else { + logs.Error("CreateScore, insert cve_score failed, score: ", score, ", err: ", err) + o.Rollback() + return 0, err + } + Id = score.Id + o.Commit() + } else { + logs.Error("CreateScore, Transaction creation failed, cveNum:", score.CveNum, ",errs: ", errs) + return 0, errs + } + return Id, nil +} + +func CreateOpenEulerSA(op *OpenEulerSA) (OpenId int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + var num int64 + if num, err = o.Insert(op); err == nil { + logs.Info("CreateOpenEulerSA, insert cve_open_euler_s_a success, num: ", num, "CveId:", op.CveId) + } else { + logs.Error("CreateOpenEulerSA, insert cve_open_euler_s_a failed, CveId: ", op.CveId, ", err:", err) + o.Rollback() + return 0, err + } + OpenId = op.OpenId + o.Commit() + } else { + logs.Error("CreateOpenEulerSA, Transaction creation failed,cveId:", op.CveId, ", errs: ", errs) + return 0, errs + } + return OpenId, nil +} + +func QueryIssueScoreRecord(cveId int64, status int8) (ScoreRecord, error) { + o := orm.NewOrm() + var sr ScoreRecord + err := o.Raw("select id, cve_id, nvd_score, n_vector_value"+ + " from cve_score_record where cve_id = ? and status = ? order by id desc limit 1", + cveId, status).QueryRow(&sr) + if err == nil { + logs.Info("QueryIssueScoreRecord, cve_score_record, search result: ", sr) + } else { + logs.Info("QueryIssueScoreRecord, cve_score_record, cveId: ", cveId, ", err: ", err) + } + return sr, err +} + +func InsertScoreRecord(sc *ScoreRecord) (scoreId int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + scoreId = 0 + if errs == nil { + if num, err := o.Insert(sc); err == nil { + logs.Info("InsertScoreRecord, insert cve_score_record, num:", num, ", CveId:", sc.CveId) + scoreId = num + } else { + logs.Error("InsertScoreRecord, insert cve_score_record failed, CveId:", sc.CveId, ",err: ", err) + o.Rollback() + return scoreId, err + } + o.Commit() + } else { + logs.Error("InsertScoreRecord, Transaction creation failed, cveId:", sc.CveId, ",errs: ", errs) + return scoreId, errs + } + return scoreId, nil +} + +func UpdateScoreRecord(sc *ScoreRecord, cols ...string) (resultNum int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + resultNum = 0 + if errs == nil { + if num, err := o.Update(sc, cols...); err == nil { + logs.Info("UpdateScoreRecord, update cve_score_record, num:", num, ", CveId:", sc.CveId) + resultNum = num + } else { + logs.Error("UpdateScoreRecord, update cve_score_record failed, CveId:", sc.CveId, ",err: ", err) + o.Rollback() + return resultNum, err + } + o.Commit() + } else { + logs.Error("InsertScoreRecord, Transaction creation failed, cveId:", sc.CveId, ",errs: ", errs) + return resultNum, errs + } + return resultNum, nil +} + +func CreateCveRelat(cve *VulnCenter, sec *SecurityNotice, score *Score, + op *OpenEulerSA, sc *ScoreRecord) (cveid int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + var num int64 + var err error + if num, err = o.Insert(cve); err == nil { + logs.Info("CreateCveRelat, insert cve_vuln_center success, num: ", num, ", cveNum: ", cve.CveNum) + } else { + logs.Error("CreateCveRelat, insert cve_vuln_center failed, CveNum: ", cve.CveNum, ",err: ", err) + o.Rollback() + return 0, err + } + sec.CveId = num + cveid = num + op.CveId = cveid + cve.CveId = num + if num, err = o.Insert(op); err == nil { + logs.Info("CreateCveRelat, insert cve_open_euler_s_a success, num: ", num, ",cveNum: ", cve.CveNum) + } else { + logs.Error("CreateCveRelat, insert cve_open_euler_s_a failed, CveNum:", cve.CveNum, ",err: ", err) + o.Rollback() + return 0, err + } + sec.OpenId = num + score.OpenId = num + if num, err = o.Insert(sec); err == nil { + logs.Info("CreateCveRelat, insert cve_security_notice success, num: ", num, ", cveNum: ", cve.CveNum) + } else { + logs.Error("CreateCveRelat, insert cve_security_notice failed, cveNum:", cve.CveNum, ",err: ", err) + o.Rollback() + return 0, err + } + score.CveId = cveid + if num, err = o.Insert(score); err == nil { + logs.Info("CreateCveRelat, insert cve_score success, num: ", num, ", cveNum: ", cve.CveNum) + } else { + logs.Error("CreateCveRelat, insert cve_score failed, CveNum:", cve.CveNum, ",err: ", err) + o.Rollback() + return 0, err + } + sc.CveId = cveid + if num, err = o.Insert(sc); err == nil { + logs.Info("CreateCveRelat, insert cve_score_record, num: ", num, ", CveId:", cve.CveId) + } else { + logs.Error("CreateCveRelat, insert cve_score_record failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return 0, err + } + o.Commit() + } else { + logs.Error("CreateCveRelat, Transaction creation failed, CveId:", cve.CveId, ",err: ", errs) + return 0, errs + } + return cveid, nil +} + +func UpdateCveRelat(cve *VulnCenter, sec *SecurityNotice, score *Score) error { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + var cv VulnCenter + cvError := o.Raw("select cve_id from cve_vuln_center where cve_id = ?", cve.CveId).QueryRow(&cv) + if cvError == nil { + var num int64 + var err error + cve.CveId = cv.CveId + if num, err = o.Update(cve); err == nil { + logs.Info("UpdateCveRelat, update cve_vuln_center success, num: ", num, ", cveNum", cve.CveNum) + } else { + logs.Error("UpdateCveRelat, update cve_vuln_center failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return err + } + } else { + var num int64 + var err error + if num, err = o.Insert(cve); err == nil { + logs.Info("UpdateCveRelat, insert cve_vuln_center success, num: ", num, ", cveNum:", cve.CveNum) + } else { + logs.Error("UpdateCveRelat, insert cve_vuln_center failed, CveNum:", cve.CveNum, ",err:", err) + o.Rollback() + return err + } + + } + var se SecurityNotice + seError := o.Raw("select sec_id from cve_security_notice where cve_id = ?", cve.CveId).QueryRow(&se) + if seError == nil { + var num int64 + sec.SecId = se.SecId + var err error + if num, err = o.Update(sec); err == nil { + logs.Info("UpdateCveRelat, update cve_security_notice success, num: ", num, ", cve.CveId", cve.CveId) + } else { + logs.Error("UpdateCveRelat, update cve_security_notice failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return err + } + } else { + sec.CveId = cve.CveId + var err error + var num int64 + if num, err = o.Insert(sec); err == nil { + logs.Info("UpdateCveRelat, insert cve_security_notice success, num: ", num, ", cveNum: ", cve.CveNum) + } else { + logs.Error("UpdateCveRelat, insert cve_security_notice failed, CveNum:", cve.CveNum, ",err:", err) + o.Rollback() + return err + } + } + var sc Score + scError := o.Raw("select id from cve_score where cve_id = ?", cve.CveId).QueryRow(&sc) + if scError == nil { + var num int64 + score.Id = sc.Id + var err error + if num, err = o.Update(score); err == nil { + logs.Info("UpdateCveRelat, update cve_score success, num: ", num, ", cve.CveId: ", cve.CveId) + } else { + logs.Error("UpdateCveRelat, update cve_score failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return err + } + } else { + score.CveId = cve.CveId + if num, err := o.Insert(score); err == nil { + logs.Info("UpdateCveRelat, insert cve_score success, num: ", num, ", cveNum:", cve.CveNum) + } else { + logs.Error("UpdateCveRelat, insert cve_score failed, CveNum: ", cve.CveNum, ",err: ", err) + o.Rollback() + return err + } + } + } else { + logs.Error("UpdateCveRelat, Transaction creation failed, CveId:", cve.CveId, ",errs: ", errs) + return errs + } + o.Commit() + return nil +} + +func UpdateCveRelat1(cve *VulnCenter, sec *SecurityNotice) (err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + v := VulnCenter{CveId: cve.CveId} + if err = o.Read(&v, "CveId"); err == nil { + var num int64 + if num, err = o.Update(cve); err == nil { + logs.Info("UpdateCveRelat1, update cve_vuln_center success, num: ", num, ", cveNum", cve.CveNum) + } else { + logs.Error("UpdateCveRelat1, update cve_vuln_center failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return err + } + } else { + var num int64 + if num, err = o.Insert(cve); err == nil { + logs.Info("UpdateCveRelat1, insert cve_vuln_center success, num: ", num, ", cveNum:", cve.CveNum) + } else { + logs.Error("UpdateCveRelat1, insert cve_vuln_center failed, CveNum:", cve.CveNum, ",err:", err) + o.Rollback() + return err + } + } + se := SecurityNotice{CveId: cve.CveId} + if err = o.Read(&se, "CveId"); err == nil { + var num int64 + if num, err = o.Update(sec); err == nil { + logs.Info("UpdateCveRelat1, update cve_security_notice success, num: ", num, ", cve.CveId:", cve.CveId) + } else { + logs.Error("UpdateCveRelat1, update cve_security_notice failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return err + } + } else { + sec.CveId = cve.CveId + if num, err := o.Insert(sec); err == nil { + logs.Info("UpdateCveRelat1, insert cve_security_notice success, num: ", num, ", cveNum:", cve.CveNum) + } else { + logs.Error("UpdateCveRelat1, insert cve_security_notice failed, CveNum:", cve.CveNum, ",err:", err) + o.Rollback() + return err + } + } + } else { + logs.Error("UpdateCveRelat1, Transaction creation failed, CveId:", cve.CveId, ",err: ", errs) + return errs + } + o.Commit() + return nil +} + +func UpdateCveRelat2(cve *VulnCenter, score *Score) (err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + v := VulnCenter{CveId: cve.CveId} + if err = o.Read(&v, "CveId"); err == nil { + var num int64 + if num, err = o.Update(cve); err == nil { + logs.Info("UpdateCveRelat2, update cve_vuln_center success, num: ", num, ", cveNum: ", cve.CveNum) + } else { + logs.Error("UpdateCveRelat2, update cve_vuln_center failed, CveId:", cve.CveId, ", err: ", err) + o.Rollback() + return err + } + } else { + var num int64 + if num, err = o.Insert(cve); err == nil { + logs.Info("UpdateCveRelat2, insert cve_vuln_center success, num: ", num, ", cveNum:", cve.CveNum) + } else { + logs.Error("UpdateCveRelat2, insert cve_vuln_center failed, CveNum:", cve.CveNum, ",err:", err) + o.Rollback() + return err + } + } + sc := Score{CveId: cve.CveId} + if err = o.Read(&sc, "CveId"); err == nil { + var num int64 + if num, err = o.Update(score); err == nil { + logs.Info("UpdateCveRelat2, update cve_score success, num: ", num, ", cve.CveId", cve.CveId) + } else { + logs.Error("UpdateCveRelat2, update cve_score failed, CveId:", cve.CveId, ",err: ", err) + o.Rollback() + return err + } + } else { + score.CveId = cve.CveId + if num, err := o.Insert(score); err == nil { + logs.Info("UpdateCveRelat2, insert cve_score success, num: ", num, ", cveNum:", cve.CveNum) + } else { + logs.Error("UpdateCveRelat2, insert cve_score failed, CveNum:", cve.CveNum, ",err: ", err) + o.Rollback() + return err + } + } + } else { + logs.Error("UpdateCveRelat2, Transaction creation failed, CveId:", cve.CveId, ",errs: ", errs) + return errs + } + o.Commit() + return nil +} + +func QueryOpenSaLastId() (OpenEulerSA, error) { + o := orm.NewOrm() + var os OpenEulerSA + err := o.Raw("select openeuler_id, openeuler_sa_num from cve_open_euler_s_a order by openeuler_id desc limit 1").QueryRow(&os) + return os, err +} + +func QueryOpenSaById(cveId int64) (OpenEulerSA, error) { + o := orm.NewOrm() + var os OpenEulerSA + err := o.Raw("select openeuler_id, openeuler_sa_num from cve_open_euler_s_a where cve_id = ?", cveId).QueryRow(&os) + return os, err +} + +func QueryOriginUpstreamRecord(our *OriginUpstreamRecord, ids, cveNum string) (error) { + o := orm.NewOrm() + err := o.Raw("select * from cve_origin_upstream_record where cve_un_ids = ? "+ + "and cve_num = ? order by cve_record_id desc limit 1", ids, cveNum).QueryRow(our) + return err +} + +func UpdateOriginUpstreamRecord(ogc *OriginUpstreamRecord, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogc, fields...) + return err +} + +func UpdateOriginStatus(updateTime, pakName, version string, cveId int64, status int) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "cve_status = ?, update_time = ? where cve_id = ? and git_packname = ? and version = ?", + status, updateTime, cveId, pakName, version).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UpdateOriginStatus, cve_origin_upstream row affected nums: ", + num, ",cveId: ", cveId, ",", updateTime, pakName, version) + return true + } + return false + } else { + logs.Error("UpdateOriginStatus, cve_origin_upstream, Update failed, cveId: ", + cveId, ",", updateTime, pakName, version, ", err: ", err) + return false + } +} + +func LockUpdateOriginStatus(updateTime, pakName, version string, cveId int64, befStatus int8, status int) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "cve_status = ?, update_time = ? where cve_id = ? and "+ + "git_packname = ? and version = ? and cve_status = ?", + status, updateTime, cveId, pakName, version, befStatus).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("LockUpdateOriginStatus, cve_origin_upstream row affected nums: ", + num, ",cveId: ", cveId, ",", updateTime, pakName, version) + return true + } + return false + } else { + logs.Error("LockUpdateOriginStatus, cve_origin_upstream, Update failed, cveId: ", + cveId, ",", updateTime, pakName, version, ", err: ", err) + return false + } +} + +func UnLockUpdateOriginStatus(beforeStatus, afterStatus int, updateTime, beforeTime string) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "cve_status = ?, update_time = ? where cve_status = ? and update_time < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateOriginStatus, cve_origin_upstream row affected nums: ", + num) + return true + } + return false + } else { + logs.Error("UnLockUpdateOriginStatus, update failed, cve_origin_upstream, err: ", err) + return false + } +} + +func UpdateOriginExcelStatus(updateTime string, cveId int64, status int) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_excel SET "+ + "cve_status = ?, update_time = ? where cve_id = ?", status, updateTime, cveId).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UpdateOriginExcelStatus, cve_origin_excel row affected nums: ", num, + ",cveId: ", cveId, ",updateTime: ", updateTime) + return true + } + return false + } else { + logs.Error("UpdateOriginExcelStatus, cve_origin_excel, cveId: ", cveId, + ",updateTime: ", updateTime, ", err: ", err) + return false + } +} + +func LockUpdateOriginExcelStatus(updateTime string, cveId int64, status int) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_excel SET "+ + "cve_status = ?, update_time = ? where cve_id = ? and cve_status in (0,1)", + status, updateTime, cveId).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("LockUpdateOriginExcelStatus, cve_origin_excel row affected nums: ", + num, ",cveId: ", cveId, ",", updateTime) + return true + } + return false + } else { + logs.Error("LockUpdateOriginExcelStatus, cve_origin_excel, cveId: ", cveId, + ",updateTime: ", updateTime, ", err: ", err) + return false + } +} + +func UnLockUpdateOriginExcelStatus(beforeStatus, afterStatus int, updateTime, beforeTime string) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_excel SET "+ + "cve_status = ?, update_time = ? where cve_status = ? and update_time < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateOriginExcelStatus, cve_origin_excel row affected nums: ", + num) + return true + } + return false + } else { + logs.Error("UnLockUpdateOriginExcelStatus, update failed, cve_origin_excel, err: ", err) + return false + } +} + +func GetSaNumCountByYears(years string, status int) int64 { + o := orm.NewOrm() + if status == 0 { + cnt, err := o.QueryTable("cve_sa_number").Filter("sa_years", years).Count() + if err == nil { + logs.Info("GetSaNumCountByYears, cve_sa_number search result：", cnt) + } else { + logs.Error("GetSaNumCountByYears, cve_sa_number, Query result failed, err: ", err) + } + return cnt + } else { + cnt, err := o.QueryTable("cve_sa_number").Filter("sa_years", years).Filter("status", status).Count() + if err == nil { + logs.Info("GetSaNumCountByYears, cve_sa_number search result：", cnt) + } else { + logs.Error("GetSaNumCountByYears, cve_sa_number, Query result failed, err: ", err, years, status) + } + return cnt + } +} + +func GetSaNumberByYears(sa *SaNumber, years string, status, orderSort int) error { + o := orm.NewOrm() + err := error(nil) + if status == 0 { + if orderSort == 1 { + err = o.Raw("select * from cve_sa_number where sa_years = ? "+ + "order by sa_id asc limit 1", years).QueryRow(&sa) + logs.Info("GetSaNumberByYears", err) + } else { + err = o.Raw("select * from cve_sa_number where sa_years = ? "+ + "order by sa_id desc limit 1", years).QueryRow(&sa) + logs.Info("GetSaNumberByYears, err: ", err) + } + } else { + if orderSort == 1 { + err = o.Raw("select * from cve_sa_number where sa_years = ? and status = ? "+ + "order by sa_id asc limit 1", years, status).QueryRow(&sa) + logs.Info("GetSaNumberByYears", err) + } else { + err = o.Raw("select * from cve_sa_number where sa_years = ? and status = ? "+ + "order by sa_id desc limit 1", years, status).QueryRow(&sa) + logs.Info("GetSaNumberByYears, err: ", err) + } + } + return err +} + +func DeleteSaNumberByYears(years string, status int) { + o := orm.NewOrm() + err := o.Raw("delete from cve_sa_number "+ + "where sa_years != ? and status = ? ", years, status).QueryRow() + logs.Info("DeleteSaNumberByYears", err) +} + +func GetSaNumber(sa *SaNumber, field ...string) error { + o := orm.NewOrm() + err := o.Read(sa, field...) + return err +} + +func InsertIssueDeleteRecord(idr *IssueDeleteRecord) error { + o := orm.NewOrm() + id, err := o.Insert(idr) + logs.Info("InsertIssueDeleteRecord, id: ", id, ", err: ", err) + return err +} + +func InsertSaNumber(sa *SaNumber) error { + o := orm.NewOrm() + id, err := o.Insert(sa) + logs.Info("InsertSaNumber, id: ", id, ", err: ", err) + return err +} + +func UpdateSaNumber(sa *SaNumber, field ...string) bool { + o := orm.NewOrm() + _, err := o.Update(sa, field...) + if err != nil { + logs.Error("UpdateSaNumber, err: ", err) + return false + } + return true +} + +func UpdateVulnCenter(cve *VulnCenter, field ...string) bool { + o := orm.NewOrm() + _, err := o.Update(cve, field...) + if err != nil { + logs.Error("UpdateVulnCenter, err: ", err) + return false + } + return true +} + +func GetVulnCenterByCid(cve *VulnCenter, field ...string) error { + o := orm.NewOrm() + err := o.Read(cve, field...) + return err +} + +func GetCanExportVulnCenterData() (data []VulnCenter, err error) { + o := orm.NewOrm() + sql := fmt.Sprintf(`SELECT * from cve_vuln_center WHERE is_export IN (0,2,3) and organizate_id = 1`) + _, err = o.Raw(sql).QueryRows(&data) + logs.Info("GetCanExportVulnCenterData, err: ", err) + return +} + +func GetCvePackageList(secId int64, affectBranch string) (data []Package, err error) { + o := orm.NewOrm() + qs := o.QueryTable("cve_package") + _, err = qs.Filter("sec_id", secId).Filter("affect_product", affectBranch).OrderBy("source").All(&data) + return +} + +func GetCanExportVulnCenterCount() (count int64) { + sql := ` SELECT COUNT(*) total FROM (SELECT COUNT(*) num,bc.cve_id,bc.cve_num + FROM (SELECT DISTINCT cve_num FROM cve_vuln_center WHERE is_export = 3) ac + JOIN cve_vuln_center bc ON ac.cve_num = bc.cve_num GROUP BY bc.cve_num) a` + res := struct { + Total int64 + }{} + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(&res) + if err != nil { + logs.Error("GetCanExportVulnCenterCount, err: ", err) + return 0 + } + return res.Total +} + +func GetCanExportTheme(cveNums, component, affectBranch string) (string, error) { + if cveNums == "" || component == "" { + return "", errors.New("param is empty") + } + s := strings.Split(cveNums, ";\n") + for k, v := range s { + s[k] = "'" + v + "'" + } + cveNums = strings.Join(s, ",") + type tmp = struct { + OpenEulerScore float64 `orm:"column(openeuler_score)"` + Theme string `orm:"column(theme)"` + AffectProduct string `orm:"column(affect_product)"` + } + res := make([]tmp, 0) + sql := fmt.Sprintf(`SELECT b.openeuler_score,a.theme,a.affect_product FROM cve_issue_template b +JOIN cve_security_notice a ON a.cve_id = b.cve_id +WHERE b.cve_num IN (%s) +AND b.owned_component = '%s'`, cveNums, component) + o := orm.NewOrm() + _, err := o.Raw(sql).QueryRows(&res) + if err != nil { + return "", err + } + max := float64(0) + resStr := "" + for _, v := range res { + if v.AffectProduct == "" || len(v.AffectProduct) < 2 { + continue + } + if max < v.OpenEulerScore { + max = v.OpenEulerScore + if affectBranch != "" && len(affectBranch) > 1 { + resStr = strings.ReplaceAll(v.Theme, v.AffectProduct, affectBranch) + } else { + resStr = v.Theme + } + } + } + return resStr, nil +} + +func GetCanExportCveData(page int64, pageSize int) (list []ExcelExport, err error) { + sql := `SELECT b.num,c.*,a.owned_component,a.cve_brief, +d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, +d.affect_product,d.reference_link,d.affect_status,e.public_date,openeuler_sa_num +FROM cve_issue_template a +RIGHT JOIN (SELECT COUNT(*) num,bc.cve_id,bc.cve_num FROM +(SELECT DISTINCT cve_num FROM cve_vuln_center WHERE is_export = 3) ac +JOIN cve_vuln_center bc ON ac.cve_num = bc.cve_num GROUP BY bc.cve_num LIMIT ?,? ) b +ON a.cve_id = b.cve_id +LEFT JOIN cve_score c +ON b.cve_id = c.cve_id +LEFT JOIN cve_security_notice d +ON b.cve_id = d.cve_id +LEFT JOIN cve_open_euler_s_a e +ON b.cve_id = e.cve_id +` + o := orm.NewOrm() + _, err = o.Raw(sql, page, pageSize).QueryRows(&list) + return +} + +//GetCanExportExcelData Get exportable data +func GetCanExportExcelData(cveNum, issueNum, repo string, issueId int64) (list []ExcelExport, err error) { + if cveNum == "" { + return list, errors.New("cve number can not empty") + } + sql := `SELECT b.num,c.*,a.issue_num,a.owned_component,a.cve_brief, +d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, +d.affect_product,d.reference_link,d.affect_status, +e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id,a.affected_version +FROM cve_issue_template a +RIGHT JOIN +(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ? AND is_export in (0,3) AND pack_name = ? AND organizate_id = 1) num, +bc.cve_id,bc.cve_num,bc.organizate_id +FROM cve_vuln_center bc WHERE bc.cve_num = ? AND bc.is_export in (0,3) AND bc.pack_name = ? AND bc.organizate_id = 1) b +ON a.cve_id = b.cve_id +LEFT JOIN cve_score c +ON b.cve_id = c.cve_id +LEFT JOIN cve_security_notice d +ON b.cve_id = d.cve_id +LEFT JOIN cve_open_euler_s_a e +ON b.cve_id = e.cve_id +WHERE a.issue_num = ? and a.issue_id = ? and b.organizate_id = 1 +` + o := orm.NewOrm() + _, err = o.Raw(sql, cveNum, repo, cveNum, repo, issueNum, issueId).QueryRows(&list) + return +} + +func GetCanExportCveDataSameNum(cId string) (list []ExcelExport, err error) { + sql := `SELECT a.cve_id,a.owned_component,a.cve_brief,c.*,d.sec_id, +d.introduction,d.summary,d.theme,d.description,d.influence_component, +d.affect_product,d.reference_link,d.affect_status, +e.public_date,e.openeuler_sa_num,a.cve_level +FROM cve_issue_template a +LEFT JOIN cve_score c +ON c.cve_id = a.cve_id +LEFT JOIN cve_security_notice d +ON d.cve_id = a.cve_id +LEFT JOIN cve_open_euler_s_a e +ON e.cve_id = a.cve_id +WHERE a.cve_id IN (%s) +ORDER BY c.openeuler_score DESC +` + sql = fmt.Sprintf(sql, cId) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&list) + return +} + +func GetIssueNumber(packName string) (issueTemp []IssueTemplate, err error) { + sql := `select * from cve_issue_template where status = 3 and issue_status = 2 and +cve_id in (select cve_id from cve_vuln_center where cve_status = 2 and +is_export in (0,3) and pack_name in ('%s') and organizate_id = 1)` + sql = fmt.Sprintf(sql, packName) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemp) + return +} + +func GetUnffectIssueNumber(startTime string) (issueTemp []IssueTemplate, err error) { + sql := `SELECT * FROM cve_issue_template WHERE STATUS = 3 AND issue_status in (2,6) AND +cve_id IN (SELECT DISTINCT cve_id FROM cve_vuln_center WHERE cve_status = 2 AND +is_export IN (0,3) and organizate_id = 1) AND create_time >= '%s'` + sql = fmt.Sprintf(sql, startTime) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemp) + return +} + +func QueryIssueCount(branchVersion string) (count int64) { + sql := "" + if len(branchVersion) > 2 { + branchVersion = "%" + branchVersion + "%" + sql = fmt.Sprintf(`SELECT COUNT(*) total FROM cve_issue_template where +affected_version like '%v' and status != 6`, branchVersion) + } else { + sql = `SELECT COUNT(*) total FROM cve_issue_template where +affected_version != '' and status != 6` + } + + res := struct { + Total int64 + }{} + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(&res) + if err != nil { + logs.Error("QueryIssueCount, err: ", err) + return 0 + } + return res.Total +} + +type CveIssueInfoData struct { + IssueNum string `orm:"column(issue_num)"` + CveNum string `orm:"column(cve_num)"` + OpeneulerScore float64 `orm:"column(openeuler_score)"` + NvdScore float64 `orm:"column(nvd_score)"` + AffectProduct string `orm:"column(affected_version)"` +} + +//QueryIssue query issuewhitelist +func QueryCveIssueByBranch(currentPage, pageSize int, milestone string) ([]CveIssueInfoData, error) { + startSize := (currentPage - 1) * pageSize + o := orm.NewOrm() + var irw []CveIssueInfoData + if len(milestone) > 2 { + milestone = "%" + milestone + "%" + num, err := o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version"+ + " FROM cve_issue_template t where t.status != 6 and t.affected_version like ? limit ? offset ?", milestone, pageSize, startSize).QueryRows(&irw) + if err == nil && num > 0 { + logs.Info("QueryCveIssueByBranch1, search num: ", num) + } else { + logs.Info("QueryCveIssueByBranch1, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return irw, err + } else { + num, err := o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version"+ + " FROM cve_issue_template t where t.status != 6 and t.affected_version != '' limit ? offset ?", pageSize, startSize).QueryRows(&irw) + if err == nil && num > 0 { + logs.Info("QueryCveIssueByBranch2, search num: ", num) + } else { + logs.Info("QueryCveIssueByBranch2, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return irw, err + } +} + +func QueryCveAllIssueCount(communityFlag int, startTime string) (count int64) { + sql := "" + if len(startTime) > 2 { + if communityFlag > 0 { + sql = fmt.Sprintf(`SELECT COUNT(t.template_id) total FROM cve_issue_template t, cve_vuln_center v where + t.status != 6 and v.organizate_id = %d and t.cve_id = v.cve_id and t.update_time >= '%s' order by t.update_time desc`, communityFlag, startTime) + } else { + sql = fmt.Sprintf(`SELECT COUNT(t.template_id) total FROM cve_issue_template t where + t.status != 6 and t.update_time >= '%s' order by t.update_time desc`, startTime) + } + } else { + if communityFlag > 0 { + sql = fmt.Sprintf(`SELECT COUNT(t.template_id) total FROM cve_issue_template t, cve_vuln_center v where + t.status != 6 and v.organizate_id = %d and t.cve_id = v.cve_id order by t.update_time desc`, communityFlag) + } else { + sql = fmt.Sprintf(`SELECT COUNT(t.template_id) total FROM cve_issue_template t where + t.status != 6 order by t.update_time desc`) + } + } + + res := struct { + Total int64 + }{} + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(&res) + if err != nil { + logs.Error("QueryCveAllIssueCount, err: ", err) + return 0 + } + return res.Total +} + +type CveAllIssueData struct { + IssueNum string `orm:"column(issue_num)"` + CveNum string `orm:"column(cve_num)"` + OpeneulerScore float64 `orm:"column(openeuler_score)"` + NvdScore float64 `orm:"column(nvd_score)"` + AffectedVersion string `orm:"column(affected_version)"` + CveLevel string `orm:"column(cve_level)"` + CreateTime time.Time `orm:"column(create_time)"` + CpublicTime string `orm:"column(repair_time)"` + FirstPerTime string `orm:"column(first_per_time)"` + FirstGetTime string `orm:"column(first_get_time)"` + RpmReleaseTime string `orm:"column(rpm_release_time)"` + SaReleaseTime string `orm:"column(sa_release_time)"` +} + +//QueryIssue query issuewhitelist +func QueryCveAllIssueData(currentPage, pageSize, communityFlag int, startTime string) ([]CveAllIssueData, error) { + startSize := (currentPage - 1) * pageSize + o := orm.NewOrm() + var cai []CveAllIssueData + if len(startTime) > 2 { + if communityFlag > 0 { + num, err := o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version,"+ + "t.cve_level,t.create_time,v.repair_time,s.sa_release_time,s.rpm_release_time,v.first_per_time, v.first_get_time"+ + " FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id "+ + "where t.status != 6 and t.update_time >= ? and t.cve_id = v.cve_id and v.organizate_id = ? "+ + "order by t.update_time desc limit ? offset ?", startTime, communityFlag, pageSize, startSize).QueryRows(&cai) + if err == nil && num > 0 { + logs.Info("QueryCveAllIssueData1, search num: ", num) + } else { + logs.Info("QueryCveAllIssueData1, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return cai, err + } else { + num, err := o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version,"+ + "t.cve_level,t.create_time,v.repair_time,s.sa_release_time,s.rpm_release_time,v.first_per_time, v.first_get_time"+ + " FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id "+ + "where t.status != 6 and t.update_time >= ? and t.cve_id = v.cve_id "+ + "order by t.update_time desc limit ? offset ?", startTime, pageSize, startSize).QueryRows(&cai) + if err == nil && num > 0 { + logs.Info("QueryCveAllIssueData2, search num: ", num) + } else { + logs.Info("QueryCveAllIssueData2, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return cai, err + } + } else { + if communityFlag > 0 { + num, err := o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version,"+ + "t.cve_level,t.create_time,v.repair_time,s.sa_release_time,s.rpm_release_time,v.first_per_time, v.first_get_time"+ + " FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id "+ + "where t.status != 6 and t.cve_id = v.cve_id and v.organizate_id = ? "+ + "order by t.update_time desc limit ? offset ?", communityFlag, pageSize, startSize).QueryRows(&cai) + if err == nil && num > 0 { + logs.Info("QueryCveAllIssueData3, search num: ", num) + } else { + logs.Info("QueryCveAllIssueData3, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return cai, err + } else { + num, err := o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version,"+ + "t.cve_level,t.create_time,v.repair_time,s.sa_release_time,s.rpm_release_time,v.first_per_time, v.first_get_time"+ + " FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id "+ + "where t.status != 6 and t.cve_id = v.cve_id "+ + "order by t.update_time desc limit ? offset ?", pageSize, startSize).QueryRows(&cai) + if err == nil && num > 0 { + logs.Info("QueryCveAllIssueData4, search num: ", num) + } else { + logs.Info("QueryCveAllIssueData4, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return cai, err + } + } +} + +func QueryCveOrgIssueCount(communityFlag int) (count int64) { + sql := "" + if communityFlag > 0 { + sql = fmt.Sprintf(`SELECT COUNT(id) total FROM cve_gite_origin_issue where organizate_id = %d order by id desc`, communityFlag) + } else { + sql = fmt.Sprintf(`SELECT COUNT(id) total FROM cve_gite_origin_issue order by id desc`) + } + res := struct { + Total int64 + }{} + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(&res) + if err != nil { + logs.Error("QueryCveOrgIssueCount, err: ", err) + return 0 + } + return res.Total +} + +//QueryIssue query issuewhitelist +func QueryCveOrgIssueData(currentPage, pageSize, communityFlag int) (goi []GiteOriginIssue) { + startSize := (currentPage - 1) * pageSize + o := orm.NewOrm() + if communityFlag > 0 { + num, err := o.Raw("SELECT * FROM cve_gite_origin_issue where organizate_id = ? order by id desc limit ? offset ?", + communityFlag, pageSize, startSize).QueryRows(&goi) + if err == nil && num > 0 { + logs.Info("QueryCveOrgIssueData1, search num: ", num) + } else { + logs.Info("QueryCveOrgIssueData1, cur_time:", + common.GetCurTime(), ",err: ", err) + } + } else { + num, err := o.Raw("SELECT * FROM cve_gite_origin_issue order by id desc limit ? offset ?", pageSize, startSize).QueryRows(&goi) + if err == nil && num > 0 { + logs.Info("QueryCveOrgIssueData2, search num: ", num) + } else { + logs.Info("QueryCveOrgIssueData2, cur_time:", + common.GetCurTime(), ",err: ", err) + } + } + return +} + +//QueryIssue query issuewhitelist +func QueryCveSingleIssueData(communityFlag int, packName, cveNum, issueNum string) (cai CveAllIssueData, err error) { + o := orm.NewOrm() + err = o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version,"+ + "t.cve_level,t.create_time,v.repair_time,s.sa_release_time,s.rpm_release_time,v.first_per_time, v.first_get_time"+ + " FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id "+ + "where t.status != 6 and t.cve_id = v.cve_id and v.organizate_id = ? "+ + "and v.pack_name=? and v.cve_num=? and t.issue_num=? limit 1", communityFlag, packName, cveNum, issueNum).QueryRow(&cai) + if err != nil { + err = o.Raw("SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num, t.affected_version,"+ + "t.cve_level,t.create_time,v.repair_time,s.sa_release_time,s.rpm_release_time,v.first_per_time, v.first_get_time"+ + " FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id "+ + "where t.status != 6 and t.cve_id = v.cve_id and v.organizate_id = ? "+ + "and v.pack_name=? and v.cve_num=? limit 1", communityFlag, packName, cveNum).QueryRow(&cai) + } + return +} diff --git a/cve-vulner-manager/models/cve_web.go b/cve-vulner-manager/models/cve_web.go new file mode 100644 index 0000000000000000000000000000000000000000..5c3c228dcab61c27fc82b6d4fd61d3671321023c --- /dev/null +++ b/cve-vulner-manager/models/cve_web.go @@ -0,0 +1,20 @@ +package models + +type RespCommon struct { + Code int + Msg string + Success bool +} + +type CveDetail struct { + Id int64 `json:"id"` + CveId string `json:"cveId"` + AffectedProduct string `json:"affectedProduct"` + UpdateTime string `json:"updateTime"` + CreateTime string `json:"announcementTime"` +} + +type RespCveDetail struct { + RespCommon + Result *CveDetail +} diff --git a/cve-vulner-manager/models/cvedetail.go b/cve-vulner-manager/models/cvedetail.go new file mode 100644 index 0000000000000000000000000000000000000000..741dc5d3904abfb37186c24153508502f1f96fa2 --- /dev/null +++ b/cve-vulner-manager/models/cvedetail.go @@ -0,0 +1,184 @@ +package models + +import ( + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +func GetOriginUpstream(cveNum string, ou *OriginUpstream) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream "+ + "WHERE cve_num=? order by cve_id desc limit 1", cveNum).QueryRow(ou) + if err == nil { + logs.Info("GetOriginUpstream, cve_origin_upstream, "+ + "cveNum: ", cveNum, ", search successful") + return nil + } else { + logs.Error("GetOriginUpstream, cve_origin_upstream, "+ + "cveNum: ", cveNum, ", Query failed") + } + return err +} + +func GetOriginDesc(cveId int64, oud *OriginUpstreamDesc) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream_desc "+ + "WHERE cve_id=?", cveId).QueryRow(oud) + if err == nil { + logs.Info("GetOriginDesc, cve_origin_upstream_desc, "+ + "cve_id: ", cveId, ", search successful") + return nil + } else { + logs.Error("GetOriginDesc, cve_origin_upstream_desc, "+ + "cve_id: ", cveId, ", Query failed") + } + return err +} + +func GetOriginConfig(cveId int64, ouc *OriginUpstreamConfig) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream_config "+ + "WHERE cve_id=?", cveId).QueryRow(ouc) + if err == nil { + logs.Info("GetOriginConfig, cve_origin_upstream_config, "+ + "cve_id: ", cveId, ", search successful") + return nil + } else { + logs.Error("GetOriginConfig, cve_origin_upstream_config, "+ + "cve_id: ", cveId, ", Query failed") + } + return err +} + +func GetOriginConfigNode(configId int64, oucn *[]OriginUpstreamConfigNode) (num int64, err error) { + o := orm.NewOrm() + num, err = o.Raw("select * FROM cve_origin_upstream_config_node "+ + "WHERE conf_id=?", configId).QueryRows(oucn) + if num > 0 && err == nil { + logs.Info("GetOriginConfigNode, cve_origin_upstream_config_node, "+ + "configId: ", configId, ", search successful") + return num, nil + } else { + logs.Error("GetOriginConfigNode, cve_origin_upstream_config_node,"+ + " configId: ", configId, ", Query failed") + } + return 0, err +} + +func GetOriginConfigNodeCpe(nodeId int64, oucnc *[]OriginUpstreamConfigNodeCpe) (num int64, err error) { + o := orm.NewOrm() + num, err = o.Raw("select * FROM cve_origin_upstream_config_node_cpe "+ + "WHERE node_id=?", nodeId).QueryRows(oucnc) + if num > 0 && err == nil { + logs.Info("GetOriginConfigNodeCpe, cve_origin_upstream_config_node_cpe,"+ + " configId: ", nodeId, ", search successful") + return num, nil + } else { + logs.Error("GetOriginConfigNodeCpe, cve_origin_upstream_config_node_cpe, "+ + "configId: ", nodeId, ", Query failed") + } + return 0, err +} + +func GetOriginPoc(cveId int64, oup *OriginUpstreamPoc) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream_poc "+ + "WHERE cve_id=?", cveId).QueryRow(oup) + if err == nil { + logs.Info("GetOriginPoc, cve_origin_upstream_poc, "+ + "cve_id: ", cveId, ", search successful") + return nil + } else { + logs.Error("GetOriginPoc, cve_origin_upstream_poc, "+ + "cve_id: ", cveId, ", Query failed") + } + return err +} + +func GetOriginEvent(cveId int64, oue *OriginUpstreamEvent) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream_event "+ + "WHERE cve_id=?", cveId).QueryRow(oue) + if err == nil { + logs.Info("GetOriginEvent, cve_origin_upstream_event, cve_id: ", cveId, ", search successful") + return nil + } else { + logs.Error("GetOriginEvent, cve_origin_upstream_event, cve_id: ", cveId, ", Query failed") + } + return err +} + +func GetOriginReference(cveId int64, our *[]OriginUpstreamReference) (num int64, err error) { + o := orm.NewOrm() + num, err = o.Raw("select * FROM cve_origin_upstream_reference "+ + "WHERE cve_id=?", cveId).QueryRows(our) + if num > 0 && err == nil { + logs.Info("GetOriginReference, cve_origin_upstream_reference, "+ + "cve_id: ", cveId, ", search successful") + return num, nil + } else { + logs.Error("GetOriginReference, cve_origin_upstream_reference, "+ + "cve_id: ", cveId, ", Query failed") + } + return 0, err +} + +func GetOriginVulType(cveId int64, ouv *OriginUpstreamVulType) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream_vul_type "+ + "WHERE cve_id=?", cveId).QueryRow(ouv) + if err == nil { + logs.Info("GetOriginVulType, cve_origin_upstream_vul_type, "+ + "cve_id: ", cveId, ", search successful") + return nil + } else { + logs.Error("GetOriginVulType, cve_origin_upstream_vul_type, "+ + "cve_id: ", cveId, ", Query failed") + } + return err +} + +func GetOriginFixSuggest(cveId int64, ouf *OriginUpstreamFixSuggest) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_origin_upstream_fix_suggest "+ + "WHERE cve_id=?", cveId).QueryRow(ouf) + if err == nil { + logs.Info("GetOriginFixSuggest, cve_origin_upstream_fix_suggest, "+ + "cve_id: ", cveId, ", search successful") + return nil + } else { + logs.Error("GetOriginFixSuggest, cve_origin_upstream_fix_suggest, "+ + "cve_id: ", cveId, ", Query failed") + } + return err +} + +func GetOriginFixSuggestRef(fixId int64, oufr *[]OriginUpstreamFixSuggestRef) (num int64, err error) { + o := orm.NewOrm() + num, err = o.Raw("select * FROM cve_origin_upstream_fix_suggest_ref "+ + "WHERE fix_id=?", fixId).QueryRows(oufr) + if num > 0 && err == nil { + logs.Info("GetOriginFixSuggestRef, cve_origin_upstream_fix_suggest_ref, "+ + "fix_id: ", fixId, ", search successful") + return num, nil + } else { + logs.Error("GetOriginFixSuggestRef, cve_origin_upstream_fix_suggest_ref, "+ + "fix_id: ", fixId, ", Query failed") + } + return 0, err +} + +func GetOriginFixSuggestRefTag(fixRefId int64, oufrt *[]OriginUpstreamFixSuggestRefTag) (num int64, err error) { + o := orm.NewOrm() + num, err = o.Raw("select * FROM cve_origin_upstream_fix_suggest_ref_tag "+ + "WHERE fix_ref_id=?", fixRefId).QueryRows(oufrt) + if num > 0 && err == nil { + logs.Info("GetOriginFixSuggestRefTag, cve_origin_upstream_fix_suggest_ref_tag, "+ + "fix_ref_id: ", fixRefId, ", search successful") + return num, nil + } else { + logs.Error("GetOriginFixSuggestRefTag, cve_origin_upstream_fix_suggest_ref_tag, "+ + "fix_ref_id: ", fixRefId, ", Query failed") + } + return 0, err +} diff --git a/cve-vulner-manager/models/excel.go b/cve-vulner-manager/models/excel.go new file mode 100644 index 0000000000000000000000000000000000000000..8b8fbe625786d54ff21c59495f964b70ec835985 --- /dev/null +++ b/cve-vulner-manager/models/excel.go @@ -0,0 +1,163 @@ +package models + +import ( + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +//ExcelExport the export excel row content model +type ExcelExport struct { + Num int64 + Score + CveBrief string `json:"cve_brief"` + OwnedComponent string `json:"owned_component"` + SecID int64 `json:"sec_id" orm:"column(sec_id)"` + Introduction string `json:"introduction"` + Summary string `json:"summary"` + Theme string `json:"theme"` + Description string `json:"description"` + InfluenceComponent string `json:"influence_component" orm:"size(256);null;column(influence_component)"` + AffectProduct string `json:"affect_product"` + ReferenceLink string `json:"reference_link"` + AffectStatus string `json:"affect_status"` + PublicDate string `json:"public_date"` + OpenEulerSANum string `json:"openeuler_sa_num" orm:"size(128);column(openeuler_sa_num)"` + CveLevel string `json:"cve_level" orm:"size(32);column(cve_level)"` + OrganizateId int8 `json:"organizate_id" orm:"column(organizate_id)"` + AffectedVersion string `json:"affected_version" orm:"column(affected_version)"` + IssueLabel string `json:"issue_label" orm:"column(issue_label)"` + Repo string `json:"repo" orm:"column(repo)"` +} + +//ExcelPackage Released packages +type ExcelPackage struct { + PubTime string + Repo string + Packages string +} + +//Insert Insert a generated excel file record +func (er ExportRecord) Insert() error { + o := orm.NewOrm() + _, err := o.Insert(&er) + return err +} + +//QueryLast query the last excel record +func (er *ExportRecord) QueryLast() error { + o := orm.NewOrm() + err := o.QueryTable(er).Filter("state", 0).OrderBy("-create_time").One(er) + return err +} + +//Update update by column name +func (er *ExportRecord) Update(field ...string) error { + o := orm.NewOrm() + _, err := o.Update(er, field...) + return err +} + +//Read read by column name +func (er *ExportRecord) Read(field ...string) error { + o := orm.NewOrm() + return o.Read(er, field...) +} + +func (elt *EmailList) Read(field ...string) ([]EmailList, error) { + o := orm.NewOrm() + var el []EmailList + var num int64 + num, err := o.Raw("select *"+ + " from cve_email_list where email_type = ?", elt.EmailType).QueryRows(&el) + if err == nil && num > 0 { + return el, nil + } + logs.Error("cve_email_list ,err: ", err) + return el, err +} + +func GetCvrfRecord(cfr *CvrfSaRecord, colName ...string) error { + o := orm.NewOrm() + err := o.Read(cfr, colName...) + return err +} + +func InsertCvrfRecord(cfr *CvrfSaRecord) (int64, error) { + o := orm.NewOrm() + num, err := o.Insert(cfr) + return num, err +} + +func UpdateCvrfRecord(cfr *CvrfSaRecord, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(cfr, fields...) + return err +} + +func GetCvrfFileName(afl *SaFileList, colName ...string) error { + o := orm.NewOrm() + err := o.Read(afl, colName...) + return err +} + +func GetCvrfAllFile() []SaFileRecord { + o := orm.NewOrm() + var afl []SaFileRecord + num, err := o.Raw("SELECT * FROM cve_sa_file_record order by file_id asc").QueryRows(&afl) + if err == nil { + logs.Info("cve_sa_file_record nums: ", num) + } else { + logs.Error("cve_sa_file_record, err: ", err) + } + return afl +} + +func InsertCvrfFileName(afl *SaFileList) (int64, error) { + o := orm.NewOrm() + num, err := o.Insert(afl) + return num, err +} + +func UpdateCvrfFileName(afl *SaFileList, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(afl, fields...) + return err +} + +func DeleteCvrfFileName(afl *SaFileList, fields ...string) error { + o := orm.NewOrm() + _, err := o.Delete(afl, fields...) + return err +} + +func DeleteCvrfFileRecord() { + o := orm.NewOrm() + err := o.Raw("delete from cve_sa_file_record ").QueryRow() + logs.Info("DeleteCvrfFileRecord", err) +} + +func InsertCvrfFileRecord(afl *SaFileRecord) (int64, error) { + o := orm.NewOrm() + num, err := o.Insert(afl) + return num, err +} + +func QueryCvrfFileRecord(afl *SaFileRecord, colName ...string) error { + o := orm.NewOrm() + err := o.Read(afl, colName...) + return err +} + +func GetCvrfSaRecordByCve(cveNum string) (afl []CvrfSaRecord) { + o := orm.NewOrm() + curCveNum := "%" + cveNum + "%" + sql := fmt.Sprintf("SELECT sort_openeuler_sa_num FROM cve_cvrf_sa_record where cve_num like '%v'", curCveNum) + num, err := o.Raw(sql).QueryRows(&afl) + if err == nil { + logs.Info("cve_cvrf_sa_record nums: ", num) + } else { + logs.Error("cve_cvrf_sa_record, err: ", err) + } + return afl +} diff --git a/cve-vulner-manager/models/gauss.go b/cve-vulner-manager/models/gauss.go new file mode 100644 index 0000000000000000000000000000000000000000..7310ed9600f143ee50db4a2075b3ebbb2760797a --- /dev/null +++ b/cve-vulner-manager/models/gauss.go @@ -0,0 +1,519 @@ +package models + +import ( + "cvevulner/common" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "strconv" + "strings" +) + +//ExcelPackage Released packages +type GaussExcelTag struct { + PubTime string + Repo string + Packages string +} + +// Provide gauss related data and mysql interface layer processing logic +func QueryDownloadFile(status int) (FileData []OpenGaussDownloadFile, err error) { + o := orm.NewOrm() + sql := fmt.Sprintf(`SELECT * from cve_open_gauss_download_file +WHERE status = %d order by repo_version desc`, status) + _, err = o.Raw(sql).QueryRows(&FileData) + logs.Info("QueryDownloadFile, err: ", err) + return +} + +func UpdateDownloadFile(fileId int64, status int) { + o := orm.NewOrm() + err := o.Raw("update cve_open_gauss_download_file set status = ?,"+ + "update_time = ? where file_id = ?", status, common.GetCurTime(), fileId).QueryRow() + logs.Info("UpdateDownloadFile", err) +} + +// Query all processed issues and filter out the unaffected data that needs to be processed +func QueryGaussIssueData(startTime string) (issueTemp []IssueTemplate, err error) { + sql := `SELECT * FROM cve_issue_template WHERE status = 3 AND issue_status in (2,6) AND +cve_id IN (SELECT DISTINCT cve_id FROM cve_vuln_center WHERE cve_status = 2 and +organizate_id = 2 and is_export != 1) AND create_time >= '%s'` + sql = fmt.Sprintf(sql, startTime) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemp) + return +} + +// Get published CVE data +func QueryReleaseGaussIssue(ogv *OpenGaussVersion, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogv, field...) + return err +} + +//GetCanExportExcelData Get exportable data +func QueryGaussExportSaData(cveNum, issueNum, repo string, issueId int64) (list []ExcelExport, err error) { + if cveNum == "" { + return list, errors.New("cve number can not empty") + } + sql := `SELECT b.num,c.*,a.issue_num,a.owned_component,a.cve_brief, +d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, +d.affect_product,d.reference_link,d.affect_status, +e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id,a.affected_version,a.issue_label,a.repo +FROM cve_issue_template a +RIGHT JOIN +(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ? AND +is_export != 1 AND pack_name = ? AND organizate_id = 2) num , +bc.cve_id,bc.cve_num,bc.organizate_id +FROM cve_vuln_center bc WHERE bc.cve_num = ? AND bc.is_export != 1 AND +bc.pack_name = ? AND bc.organizate_id = 2) b +ON a.cve_id = b.cve_id +LEFT JOIN cve_score c +ON b.cve_id = c.cve_id +LEFT JOIN cve_security_notice d +ON b.cve_id = d.cve_id +LEFT JOIN cve_open_euler_s_a e +ON b.cve_id = e.cve_id +WHERE a.issue_num = ? and a.issue_id = ? and b.organizate_id = 2 +` + o := orm.NewOrm() + _, err = o.Raw(sql, cveNum, repo, cveNum, repo, issueNum, issueId).QueryRows(&list) + return +} + +func QueryReleaseCve(ogc *OpenGaussCveList, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogc, field...) + return err +} + +// update data +func UpdateReleaseCve(ogc *OpenGaussCveList, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogc, fields...) + return err +} + +// insert data +func InsertReleaseCve(ogc *OpenGaussCveList) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogc) + return id, err +} + +func QueryGaussVersion(ogv *OpenGaussVersion, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogv, field...) + return err +} + +func UpdateGaussVersion(ogv *OpenGaussVersion, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogv, fields...) + return err +} + +// insert data +func InsertGaussVersion(ogv *OpenGaussVersion) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogv) + return id, err +} + +func GetGaussIssueNumber(packName string) (issueTemp []IssueTemplate, err error) { + sql := `select * from cve_issue_template where status = 3 and issue_status = 2 and +cve_id in (select cve_id from cve_vuln_center where cve_status = 2 and +is_export in (0,3) and repo_name in ('%s') and organizate_id = 2)` + sql = fmt.Sprintf(sql, packName) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemp) + return +} + +func QuerySaDataByPackName(ogl *OpenGaussListTemp) error { + sql := `select * from cve_open_gauss_list_temp where influence_component = '%s' and affect_product = '%s'` + sql = fmt.Sprintf(sql, ogl.InfluenceComponent, ogl.AffectProduct) + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(ogl) + if ogl.GaussTempId > 0 { + return nil + } else { + return err + } +} + +func UpdateSaData(ogl *OpenGaussListTemp, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogl, fields...) + return err +} + +// insert data +func InsertSaData(ogl *OpenGaussListTemp) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogl) + return id, err +} + +func GetGaussExportThem(cveNums, component, affectBranch string) (string, error) { + if cveNums == "" || component == "" { + return "", errors.New("param is empty") + } + s := strings.Split(cveNums, ";") + for k, v := range s { + s[k] = "'" + v + "'" + } + cveNums = strings.Join(s, ",") + type tmp = struct { + OpenEulerScore float64 `orm:"column(openeuler_score)"` + Theme string `orm:"column(theme)"` + AffectProduct string `orm:"column(affect_product)"` + } + res := make([]tmp, 0) + sql := fmt.Sprintf(`SELECT b.openeuler_score,a.theme,a.affect_product FROM cve_issue_template b +JOIN cve_security_notice a ON a.cve_id = b.cve_id +WHERE b.cve_num IN (%s) +AND b.owned_component = '%s'`, cveNums, component) + o := orm.NewOrm() + _, err := o.Raw(sql).QueryRows(&res) + if err != nil { + return "", err + } + max := float64(0) + resStr := "" + for _, v := range res { + if v.AffectProduct == "" || len(v.AffectProduct) < 2 { + continue + } + if max < v.OpenEulerScore { + max = v.OpenEulerScore + if affectBranch != "" && len(affectBranch) > 1 { + resStr = strings.ReplaceAll(v.Theme, v.AffectProduct, affectBranch) + } else { + resStr = v.Theme + } + } + } + return resStr, nil +} + +func QueryOpenGaussGroup(ogg *OpenGaussGroup, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogg, field...) + return err +} + +// insert data +func InsertOpenGaussGroup(ogg *OpenGaussGroup) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogg) + return id, err +} + +func QueryOpenGaussPlatform(ogp *OpenGaussPlatform, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogp, field...) + return err +} + +// insert data +func InsertOpenGaussPlatform(ogp *OpenGaussPlatform) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogp) + return id, err +} + +func QueryOpenGaussPackage(ogk *OpenGaussPackage, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogk, field...) + return err +} + +// insert data +func InsertOpenGaussPackage(ogk *OpenGaussPackage) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogk) + return id, err +} + +func UpdateOpenGaussPackage(ogk *OpenGaussPackage, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogk, fields...) + return err +} + +func QueryGaussSaMaxValue(ogs *OpenGaussSiteList) (err error) { + sql := `select * from cve_open_gauss_site_list where status = 3 order by gauss_id desc limit 1` + o := orm.NewOrm() + err = o.Raw(sql).QueryRow(ogs) + return +} + +func QueryOpenGaussSitePreAll() (ogl []OpenGaussSiteList, err error) { + sql := `select * from cve_open_gauss_site_list where status != 3` + o := orm.NewOrm() + num, err := o.Raw(sql).QueryRows(&ogl) + logs.Info("QueryOpenGaussSitePreAll, err: ", err, num) + return +} + +func QuerySaTempByPackName(packName string) (ogl []OpenGaussListTemp, err error) { + sql := `select * from cve_open_gauss_list_temp where influence_component = '%s' and status = 1` + sql = fmt.Sprintf(sql, packName) + o := orm.NewOrm() + num, err := o.Raw(sql).QueryRows(&ogl) + logs.Info("QuerySaTempByPackName, err: ", err, num) + return +} + +func DeleteSaTempByPackName(packName string) { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_list_temp where influence_component = ?", packName).QueryRow() + logs.Info("DeleteSaTempByPackName", err) +} + +func QueryOpenGaussCveByGid(gaussId int64) (ogl []OpenGaussCveList, err error) { + sql := "" + if gaussId == 0 { + sql = fmt.Sprintf(`select * from cve_open_gauss_cve_list where status != 3`) + } else { + sql = fmt.Sprintf(`select * from cve_open_gauss_cve_list where gauss_id = %d and status != 3`, gaussId) + } + o := orm.NewOrm() + num, err := o.Raw(sql).QueryRows(&ogl) + logs.Info("QueryOpenGaussCveByGid, err: ", err, num) + return +} + +func DeleteOpenGaussVersionById(cveId int64) { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_version where cve_id = ?", cveId).QueryRow() + logs.Info("DeleteOpenGaussVersionById", err) +} + +func DeleteGaussVersionByIdStatus(cveId int64) { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_version where cve_id = ? and release_flag = 1", cveId).QueryRow() + logs.Info("DeleteGaussVersionByIdStatus", err) +} + +func DeleteOpenGaussCveByGid(gaussId int64) { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_cve_list where gauss_id = ?", gaussId).QueryRow() + logs.Info("DeleteOpenGaussCveById", err) +} + +func DeleteOpenGaussCveByCid(cveId int64) { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_cve_list where id = ?", cveId).QueryRow() + logs.Info("DeleteOpenGaussCveByCid", err) +} + +func DeleteOpenGaussPackByGid(gaussId int64) { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_package where gauss_id = ?", gaussId).QueryRow() + logs.Info("DeleteOpenGaussPackByGid", err) +} + +func DeleteOpenGaussSitePreAll() { + o := orm.NewOrm() + err := o.Raw("delete from cve_open_gauss_site_list where status != 3").QueryRow() + logs.Info("DeleteOpenGaussSitePreAll", err) +} + +// insert data +func InsertOpenGaussSiteList(osl *OpenGaussSiteList) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(osl) + return id, err +} + +func UpdateOpenGaussCveByGid(gaussId, sGaussId int64) { + o := orm.NewOrm() + err := o.Raw("update cve_open_gauss_cve_list set gauss_id =?,update_time = ? where gauss_id = ?", + sGaussId, common.GetCurTime(), gaussId).QueryRow() + logs.Info("UpdateOpenGaussCveByGid", err) +} + +func UpdateOpenGaussVersionByCid(tempCveId, cveId int64, cveNum string) { + o := orm.NewOrm() + err := o.Raw("update cve_open_gauss_version set cve_id =?,update_time = ? where cve_id = ? and cve_num = ?", + cveId, common.GetCurTime(), tempCveId, cveNum).QueryRow() + logs.Info("UpdateOpenGaussVersionByCid", err) +} + +func UpdateOpenGaussPackByGid(gaussId, sGaussId int64) { + o := orm.NewOrm() + err := o.Raw("update cve_open_gauss_package set gauss_id = ?,update_time = ? where gauss_id = ?", + sGaussId, common.GetCurTime(), gaussId).QueryRow() + logs.Info("UpdateOpenGaussPackByGid", err) +} + +// Upgrade pre-release data to official data +func UpdateGaussPreTRelease() { + o := orm.NewOrm() + siteErr := o.Raw("update cve_open_gauss_site_list set status = 3,update_time = ? where status = 1", + common.GetCurTime()).QueryRow() + if siteErr != nil { + logs.Error("UpdateGaussPreTRelease, update cve_open_gauss_site_list failed ,err: ", siteErr) + } + o = orm.NewOrm() + cveErr := o.Raw("update cve_open_gauss_cve_list set status = 3,update_time = ? where status = 1", + common.GetCurTime()).QueryRow() + if cveErr != nil { + logs.Error("UpdateGaussPreTRelease, update cve_open_gauss_cve_list failed ,err: ", cveErr) + } +} + +func QueryGaussSaCount(cveLevel, years, releaseFlag int, searchName string) (count int64) { + conditStr := SearchGaussCondSa(cveLevel, years, releaseFlag, searchName) + sql := "" + if len(conditStr) > 2 { + sql = fmt.Sprintf(`SELECT COUNT(gauss_id) total FROM cve_open_gauss_site_list where %v`, conditStr) + } else { + sql = fmt.Sprintf(`SELECT COUNT(gauss_id) total FROM cve_open_gauss_site_list`) + } + + res := struct { + Total int64 + }{} + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(&res) + if err != nil { + logs.Error("QueryGaussSaCount, err: ", err) + return 0 + } + return res.Total +} + +//QueryIssue query QueryOpenGaussSiteList +func QueryOpenGaussSiteList(pageNum, pageSize, cveLevel, years, + releaseFlag int, searchName string) ([]OpenGaussSiteList, error) { + conditStr := SearchGaussCondSa(cveLevel, years, releaseFlag, searchName) + startSize := (pageNum - 1) * pageSize + o := orm.NewOrm() + var psl []OpenGaussSiteList + if len(conditStr) > 2 { + sql := fmt.Sprintf(`SELECT * from cve_open_gauss_site_list where %s +order by gauss_id desc limit %d offset %d`, conditStr, pageSize, startSize) + logs.Info("sql===> ", sql) + num, err := o.Raw(sql).QueryRows(&psl) + if err == nil && num > 0 { + logs.Info("QueryOpenGaussSiteList, search num: ", num) + } else { + logs.Info("QueryOpenGaussSiteList, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return psl, err + } else { + sql := fmt.Sprintf(`SELECT * from cve_open_gauss_site_list +order by gauss_id desc limit %d offset %d`, pageSize, startSize) + num, err := o.Raw(sql).QueryRows(&psl) + if err == nil && num > 0 { + logs.Info("QueryOpenGaussSiteList, search num: ", num) + } else { + logs.Info("QueryOpenGaussSiteList, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return psl, err + } +} + +func QueryOpenGaussSaByNum(osl *OpenGaussSiteList, field ...string) error { + o := orm.NewOrm() + err := o.Read(osl, field...) + return err +} + +func QueryGaussPackageByGid(gaussId int64, repoVersion string) (ogl []OpenGaussPackage, err error) { + sql := `select * from cve_open_gauss_package where gauss_id = %d and repo_version = '%s' and status = 1` + sql = fmt.Sprintf(sql, gaussId, repoVersion) + o := orm.NewOrm() + num, err := o.Raw(sql).QueryRows(&ogl) + logs.Info("QueryGaussPackageByGid, err: ", err, num) + return +} + +func QueryGaussPlatByPids(platIds []int64) (ogl []OpenGaussPlatform, err error) { + platStr := "(" + if len(platIds) > 0 { + for _, pid := range platIds { + platStr = platStr + strconv.FormatInt(pid, 10) + "," + } + platStr = platStr[:len(platStr)-1] + ")" + sql := fmt.Sprintf(`select * from cve_open_gauss_platform where plat_id in %s and status = 1`, platStr) + o := orm.NewOrm() + num, err := o.Raw(sql).QueryRows(&ogl) + logs.Info("QueryGaussPackageByGid, err: ", err, num) + } + return +} + +func QueryGaussCveCount(releaseFlag, isAffectFlag int, searchName string) (count int64) { + conditStr := SearchGaussCondCve(releaseFlag, isAffectFlag, searchName) + sql := "" + if len(conditStr) > 2 { + sql = fmt.Sprintf(`SELECT COUNT(id) total FROM cve_open_gauss_cve_list where %v`, conditStr) + } else { + sql = fmt.Sprintf(`SELECT COUNT(id) total FROM cve_open_gauss_cve_list`) + } + + res := struct { + Total int64 + }{} + o := orm.NewOrm() + err := o.Raw(sql).QueryRow(&res) + if err != nil { + logs.Error("QueryGaussCveCount, err: ", err) + return 0 + } + return res.Total +} + +// query QueryOpenGaussCveList +func QueryOpenGaussCveList(pageNum, pageSize, releaseFlag, + isAffectFlag int, searchName string) ([]OpenGaussCveList, error) { + conditStr := SearchGaussCondCve(releaseFlag, isAffectFlag, searchName) + startSize := (pageNum - 1) * pageSize + o := orm.NewOrm() + var psl []OpenGaussCveList + if len(conditStr) > 2 { + sql := fmt.Sprintf(`SELECT * from cve_open_gauss_cve_list +where %s order by id desc limit %d offset %d`, conditStr, pageSize, startSize) + num, err := o.Raw(sql).QueryRows(&psl) + if err == nil && num > 0 { + logs.Info("QueryOpenGaussCveList, search num: ", num) + } else { + logs.Info("QueryOpenGaussCveList, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return psl, err + } else { + sql := fmt.Sprintf(`SELECT * from cve_open_gauss_cve_list order by id desc limit %d offset %d`, + pageSize, startSize) + num, err := o.Raw(sql).QueryRows(&psl) + if err == nil && num > 0 { + logs.Info("QueryOpenGaussCveList2, search num: ", num) + } else { + logs.Info("QueryOpenGaussCveList2, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return psl, err + } +} + +func QueryOpenGaussCveByNum(osl *OpenGaussCveList, field ...string) error { + o := orm.NewOrm() + err := o.Read(osl, field...) + return err +} + +func QueryGaussVersionByCid(cveId int64) (ogv []OpenGaussVersion, err error) { + sql := `select * from cve_open_gauss_version where cve_id = %d order by repo_version desc` + sql = fmt.Sprintf(sql, cveId) + o := orm.NewOrm() + num, err := o.Raw(sql).QueryRows(&ogv) + logs.Info("QueryGaussVersionByCid, err: ", err, num) + return +} diff --git a/cve-vulner-manager/models/giteeissue.go b/cve-vulner-manager/models/giteeissue.go new file mode 100644 index 0000000000000000000000000000000000000000..2ac700fde84d0d99a405ed587bf35750d62d2ab5 --- /dev/null +++ b/cve-vulner-manager/models/giteeissue.go @@ -0,0 +1,484 @@ +package models + +import ( + "cvevulner/common" + "cvevulner/util" + "errors" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "strings" +) + +//Loophole issue body model +type Loophole struct { + Number string //编号 + Components string // 漏洞归属的组件 + Version string // 漏洞归属的版本 + CvsScore string //cvss 评分 + CvsVector string //cvs向量 + BriefIntroduction string //漏洞简述 + Influences string //漏洞造成的影响 + Principle string //漏洞原理 + OpScore string //openEuler 评分 + OpVector string //openEuler 向量 + InfVersion string //受影响的组件版本 + AvoidScheme string //规避方案 + InfPackage string //受影响的软件包 + InfProduct string //受影响的产品 + Repo string //漏洞所属仓库 + RepoDesc string //组件描述 + ScoreType string //评分类型 v2 v3 + AbiVersion string // abi变化 +} + +//VectorValue cvss V3.0 vector model +type VectorValue struct { + AttackVector string //AV + AttackComplexity string //AC + PrivilegesRequired string //PR + UserInteraction string //UI + Scope string //S + Confidentiality string //C + Integrity string //I + Availability string //A +} + +//VectorValueV2 cvss v2 vector model +type VectorValueV2 struct { + AccessVector string //AV + AccessComplexity string //AC + Authentication string //Au + Confidentiality string //C + IntegrityImpact string //I + Availability string //A +} + +func (g *GiteOriginIssue) Insert() error { + o := orm.NewOrm() + _, err := o.Insert(g) + return err +} + +func (g GiteOriginIssue) Update() error { + o := orm.NewOrm() + _, err := o.Update(g) + return err +} + +func (g *GiteOriginIssue) InsertOrUpdate(flag int) error { + if g.Number == "" { + return errors.New("issue number can not be null") + } + o := orm.NewOrm() + if flag == 1 { + ex := GiteOriginIssue{Number: g.Number, CveNumber: g.CveNumber, RepoPath: g.RepoPath, IssueId: g.IssueId, Owner: g.Owner} + err := o.Read(&ex, "number", "cve_number", "repo_path", "issue_id") + if err == nil && ex.Id > 0 { + g.Id = ex.Id + } + logs.Info("processing, repo: ", g.RepoPath, g.Number, g.CveNumber, g.Owner) + _, err = o.InsertOrUpdate(g) + return err + } else { + list := []GiteOriginIssue{} + num, err := o.QueryTable("cve_gite_origin_issue").Filter("cve_number", g.CveNumber). + Filter("repo_path", g.RepoPath).Filter("owner", g.Owner).All(&list) + //ex := GiteOriginIssue{CveNumber: g.CveNumber, RepoPath: g.RepoPath, Owner: g.Owner} + //err := o.Read(&ex, "cve_number", "repo_path") + if err == nil && len(list) > 0 { + for _, li := range list { + g.Id = li.Id + } + + } + logs.Info("num: ", num, ", processing, repo: ", g.RepoPath, g.Number, g.CveNumber, g.Owner) + _, err = o.InsertOrUpdate(g) + return err + } +} + +func (g *GiteOriginIssue) Detlete() error { + if g.Number == "" { + return errors.New("issue number can not be null") + } + o := orm.NewOrm() + ex := GiteOriginIssue{Number: g.Number, CveNumber: g.CveNumber, RepoPath: g.RepoPath, IssueId: g.IssueId} + num, err := o.Delete(&ex, "number", "cve_number", "repo_path", "issue_id") + if err == nil && num > 0 { + logs.Info("GiteOriginIssue, successfully deleted, "+ + "Number: ", g.Number, ",CveNumber:", g.CveNumber, ",RepoPath:", g.RepoPath, ",IssueId:", g.IssueId) + } + return err +} + +func UpdateCveCenterStatus(cveId int64, cveNum string, status int8) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_vuln_center SET cve_status = ? WHERE cve_id = ? and cve_num = ?", + status, cveId, cveNum).QueryRow() + return +} + +func GetGiteOriginIssue(cveNum string) (oi GiteOriginIssue, err error) { + o := orm.NewOrm() + oi.CveNumber = cveNum + err = o.Read(&oi, "cve_number") + return +} + +func (g *GiteOriginIssue) ParseToLoophole() (hole Loophole, err error) { + lp := Loophole{} + if g.Body == "" { + return lp, errors.New("can not parse a null body") + } + lp.InfProduct = g.InfProduct + lp.Repo = g.RepoPath + lp.RepoDesc = g.RepoDesc + scoreType, err := judgeScoreType(g.Body) + if err != nil { + logs.Error(err, "judgeScoreType, body: ", g.Body) + //return lp, err + } + lp.ScoreType = scoreType + if isNewTpl(g.Body) { + parseNewTplToLoopHole(&lp, g.Body) + } else { + parseOldTplToLoopHole(&lp, g.Body) + } + return lp, nil +} + +func (v *VectorValue) VctToVectorValue(vector string) bool { + if util.TrimString(vector) == "" { + return false + } + m, ok := util.VctToMap(vector) + if !ok { + return false + } + v.AttackVector = util.ReadVMValue(m["AV"]) + v.AttackComplexity = util.ReadVMValue(m["AC"]) + v.Availability = util.ReadVMValue(m["A"]) + v.Confidentiality = util.ReadVMValue(m["C"]) + v.Integrity = util.ReadVMValue(m["I"]) + v.PrivilegesRequired = util.ReadVMValue(m["PR"]) + v.UserInteraction = util.ReadVMValue(m["UI"]) + v.Scope = util.ReadVMValue(m["S"]) + return true +} + +func (v *VectorValueV2) VctToVectorValue(vector string) bool { + if util.TrimString(vector) == "" { + return false + } + m, ok := util.VctToMap(vector) + if !ok { + return false + } + v.AccessVector = util.ReadVMValueV2(m["AV"]) + v.AccessComplexity = util.ReadVMValueV2(m["AC"]) + v.Availability = util.ReadVMValueV2(m["A"]) + v.Confidentiality = util.ReadVMValueV2(m["C"]) + v.IntegrityImpact = util.ReadVMValueV2(m["I"]) + v.Authentication = util.ReadVMValueV2(m["Au"]) + return true +} + +func judgeScoreType(body string) (st string, err error) { + if body == "" { + return "", errors.New("can not judge score type by nil body") + } + body = strings.ReplaceAll(body, " ", "") + tb := []byte(body) + vs := util.RegexpScoreTypeV2.Find(tb) + if len(vs) > 0 { + return util.CvsScoreV2, nil + } + vs = util.RegexpScoreTypeV3.Find(tb) + if len(vs) > 0 { + return util.CvsScoreV3, nil + } + return "", errors.New("can not judge score type") +} + +func isNewTpl(body string) bool { + tb := []byte(body) + v1 := util.RegexpIsNewTpl.Find(tb) + v2 := util.RegexpIsNewTpl2.Find(tb) + if len(v1) > 0 || len(v2) > 0 { + return false + } else { + return true + } +} + +func parseOldTplToLoopHole(lp *Loophole, body string) { + sm := util.RegexpCveNumber.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + val := sm[0][1] + lp.Number = util.GetCveNumber(util.TrimString(val)) + } + sm = util.RegexpCveComponents.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 1 { + for _, v := range sm[0][1:] { + if v != "" { + lp.Components = util.TrimString(v) + break + } + } + } + sm = util.RegexpCveVersion.FindAllStringSubmatch(body, 1) + if len(sm) > 0 && len(sm[0]) > 1 { + for _, v := range sm[0][1:] { + if v != "" { + lp.Version = util.TrimString(v) + break + } + } + } + sm = util.RegexpCveScore.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + vs := util.TrimString(sm[0][1]) + if cs := util.ExtractDigital(vs); cs != "" { + lp.CvsScore = cs + } + if v := util.ExtractVector(vs, lp.ScoreType); v != "" { + lp.CvsVector = v + } + } + sm = util.RegexpCveBriefDesc.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.BriefIntroduction = sm[0][1] + } + sm = util.RegexpCveInfluencesDesc.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.Influences = sm[0][1] + } + sm = util.RegexpCvePrincipleDesc.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.Principle = sm[0][1] + } + sm = util.RegexpCveOpScore.FindAllStringSubmatch(body, -1) + if len(sm) > 0 { + vp := util.TrimString(sm[0][1]) + if cs := util.ExtractDigital(vp); cs != "" { + lp.OpScore = cs + } + if v := util.ExtractVector(vp, lp.ScoreType); v != "" { + lp.OpVector = v + } + } + sm = util.RegexpCveInfluencesVersion.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.InfVersion = util.TrimString(sm[0][1]) + infVersionSlice := strings.Split(lp.InfVersion, ",") + if len(infVersionSlice) > 0 { + infTmpSlice := []string{} + for _, ise := range infVersionSlice { + infTmpSlice = append(infTmpSlice, common.BranchVersionRep(ise)) + } + if len(infTmpSlice) > 0 { + lp.InfVersion = strings.Join(infTmpSlice, ",") + } + } + } + sm = util.RegexpCvePlannedMeasures.FindAllStringSubmatch(body, -1) + if len(sm) > 0 { + if len(sm[0]) > 0 { + lp.AvoidScheme = sm[0][1] + } + + } else { + sm = util.RegexpCvePlannedMeasures1.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.AvoidScheme = sm[0][1] + } + } + sm = util.RegexpCveInfluencesPkg.FindAllStringSubmatch(body, -1) + if len(sm) > 0 { + lp.InfPackage = util.TrimString(sm[0][1]) + } + if lp.OpVector == "" { + if lp.OpScore == "" { + lp.OpScore = lp.CvsScore + lp.OpVector = lp.CvsVector + } else { + if lp.OpScore == lp.CvsScore { + lp.OpVector = lp.CvsVector + } + } + } +} + +func parseNewTplToLoopHole(lp *Loophole, body string) { + sm := util.RegexpCveNumber.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + val := sm[0][1] + lp.Number = util.GetCveNumber(util.TrimString(val)) + } + sm = util.RegexpCveComponents.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 1 { + for _, v := range sm[0][1:] { + if v != "" { + lp.Components = util.TrimString(v) + break + } + } + } + sm = util.RegexpCveVersion.FindAllStringSubmatch(body, 1) + if len(sm) > 0 && len(sm[0]) > 1 { + for _, v := range sm[0][1:] { + if v != "" { + lp.Version = util.TrimString(v) + break + } + } + } + sm = util.RegexpCveScore.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + vs := util.TrimString(sm[0][1]) + if cs := util.ExtractDigital(vs); cs != "" { + lp.CvsScore = cs + } + if v := util.ExtractVector(vs, lp.ScoreType); v != "" { + lp.CvsVector = v + } + } + sm = util.RegexpCveBriefDescNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.BriefIntroduction = strings.Replace(sm[0][1], "二、漏洞分析结构反馈", "", -1) + } + sm = util.RegexpCveInfluencesDescNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + lp.Influences = sm[0][1] + } + sm = util.RegexpCveOpScoreNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 { + vp := util.TrimString(sm[0][1]) + if cs := util.ExtractDigital(vp); cs != "" { + lp.OpScore = cs + } + if v := util.ExtractVector(vp, lp.ScoreType); v != "" { + lp.OpVector = v + } + } + body = strings.ReplaceAll(body, " ", "") + if IsAbiInfo(body) { + if IsFixMatching(body) { + sm = util.RegexpCveInfluencesVersionNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + value := util.ExtractCommentEffectVersion(sm[0][1]) + infVersion := util.TrimString(value) + lp.InfVersion = util.TrimString(value) + infVersionSlice := strings.Split(infVersion, ",") + if len(infVersionSlice) > 0 { + infTmpSlice := []string{} + for _, ise := range infVersionSlice { + infTmpSlice = append(infTmpSlice, common.BranchVersionRep(ise)) + } + if len(infTmpSlice) > 0 { + lp.InfVersion = strings.Join(infTmpSlice, ",") + } + } + } + } else { + sm = util.RegexpCveInfluencesVersionFixNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + value := util.ExtractCommentEffectVersion(sm[0][1]) + infVersion := util.TrimString(value) + lp.InfVersion = util.TrimString(value) + infVersionSlice := strings.Split(infVersion, ",") + if len(infVersionSlice) > 0 { + infTmpSlice := []string{} + for _, ise := range infVersionSlice { + infTmpSlice = append(infTmpSlice, common.BranchVersionRep(ise)) + } + if len(infTmpSlice) > 0 { + lp.InfVersion = strings.Join(infTmpSlice, ",") + } + } + } + } + } else { + sm = util.RegexpCveAbiVersionNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + value := util.ExtractCommentEffectVersion(sm[0][1]) + infVersion := util.TrimString(value) + lp.InfVersion = util.TrimString(value) + infVersionSlice := strings.Split(infVersion, ",") + if len(infVersionSlice) > 0 { + infTmpSlice := []string{} + for _, ise := range infVersionSlice { + infTmpSlice = append(infTmpSlice, common.BranchVersionRep(ise)) + } + if len(infTmpSlice) > 0 { + lp.InfVersion = strings.Join(infTmpSlice, ",") + } + } + } + if IsFixMatching(body) { + sm = util.RegexpCveAbiNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + value := util.ExtractCommentAbiVersion(sm[0][1]) + infVersion := util.TrimString(value) + lp.AbiVersion = util.TrimString(value) + infVersionSlice := strings.Split(infVersion, ",") + if len(infVersionSlice) > 0 { + infTmpSlice := []string{} + for _, ise := range infVersionSlice { + infTmpSlice = append(infTmpSlice, common.BranchVersionRep(ise)) + } + if len(infTmpSlice) > 0 { + lp.AbiVersion = strings.Join(infTmpSlice, ",") + } + } + } + } else { + sm = util.RegexpCveAbiFixNew.FindAllStringSubmatch(body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + value := util.ExtractCommentAbiVersion(sm[0][1]) + infVersion := util.TrimString(value) + lp.AbiVersion = util.TrimString(value) + infVersionSlice := strings.Split(infVersion, ",") + if len(infVersionSlice) > 0 { + infTmpSlice := []string{} + for _, ise := range infVersionSlice { + infTmpSlice = append(infTmpSlice, common.BranchVersionRep(ise)) + } + if len(infTmpSlice) > 0 { + lp.AbiVersion = strings.Join(infTmpSlice, ",") + } + } + } + } + } + if lp.OpVector == "" { + if lp.OpScore == "" { + lp.OpScore = lp.CvsScore + lp.OpVector = lp.CvsVector + } else { + if lp.OpScore == lp.CvsScore { + lp.OpVector = lp.CvsVector + } + } + } +} + +func IsAbiInfo(body string) bool { + tb := []byte(body) + v1 := util.RegexpIsAbiTpl.Find(tb) + if len(v1) > 0 { + return false + } else { + return true + } +} + +func IsFixMatching(body string) bool { + tb := []byte(body) + v1 := util.RegexpIsFixTpl.Find(tb) + if len(v1) > 0 { + return false + } else { + return true + } +} diff --git a/cve-vulner-manager/models/hookevent.go b/cve-vulner-manager/models/hookevent.go new file mode 100644 index 0000000000000000000000000000000000000000..318cc98324cba8c7726977531d8646b8530b0e32 --- /dev/null +++ b/cve-vulner-manager/models/hookevent.go @@ -0,0 +1,207 @@ +package models + +import ( + "strings" + "time" +) + +//HookUser gitee user model +type HookUser struct { + Id int64 + Login string //Same as username + Name string //User's nickname + Email string //User's mailbox + UserName string `json:"username"` //User's code cloud personal space address + AvatarUrl string `json:"avatar_url"` //profile picture + SiteAdmin bool `json:"site_admin"` //Is it an administrator + +} + +//HookIssue gitee issue model +type HookIssue struct { + Id int64 + Number string + Title string + Labels []IssueLabel `json:"labels"` + State string `json:"state"` + HtmlUrl string `json:"html_url"` //Comment on the url on the code cloud + Body string `json:"body"` + StateName string `json:"state_name"` + User HookUser + Assignee HookUser + Repository HookRepository + CreateAt time.Time `json:"created_at"` + UpdateAt time.Time `json:"updated_at"` + FinishedAt time.Time `json:"finished_at"` + IssueType string `json:"issue_type"` + TypeName string `json:"type_name"` + SecurityHole bool `json:"security_hole"` + IssueState string `json:"issue_state"` +} + +//IssueLabel issue label +type IssueLabel struct { + Id int64 `json:"id"` + Name string `json:"name"` + Color string `json:"color"` +} + +//HookRepository gitee repository model +type HookRepository struct { + Id int64 + FullName string `json:"full_name"` + Url string `json:"url"` + Path string `json:"path"` + Name string + Owner HookUser + Private bool + public bool +} + +//HookIssueRepository gitee repository model +type HookIssueRepository struct { + Id int64 + FullName string `json:"full_name"` + Url string `json:"url"` + Path string `json:"path"` + NameSpace string `json:"namespace"` + Name string + Owner HookUser + Private bool + public bool +} + +//GitIssue gitee issue model +type GitIssue struct { + Id int64 + Number string + Title string + Labels []IssueLabel `json:"labels"` + State string `json:"state"` + HtmlUrl string `json:"html_url"` //Comment on the url on the code cloud + Body string `json:"body"` + StateName string `json:"state_name"` + User HookUser + Assignee HookUser + Repository Repository + CreateAt time.Time `json:"created_at"` + UpdateAt time.Time `json:"updated_at"` + FinishedAt time.Time `json:"finished_at"` + IssueType string `json:"issue_type"` + TypeName string `json:"type_name"` + SecurityHole bool `json:"security_hole"` + IssueState string `json:"issue_state"` +} + +//Repository gitee repository model +type Repository struct { + Id int64 + FullName string `json:"full_name"` + Url string `json:"url"` + Path string `json:"path"` + Name string + NameSpace NameSpace `json:"namespace"` + Owner HookUser + Private bool + Public bool +} + +type NameSpace struct { + Id int64 + Path string `json:"path"` + Name string `json:"name"` +} + +//HookComment gitee comment model +type HookComment struct { + User *HookUser //Comment author information + HtmlUrl string `json:"html_url"` //Comment on the url on the code cloud + Id int64 + Body string `json:"body"` //Comment content + CreateAt time.Time `json:"created_at"` + UpdateAt time.Time `json:"updated_at"` +} + +//CommentPayload hook data triggered by a comment task operation +type CommentPayload struct { + Action string // Action comment + HookName string `json:"hook_name"` + Password string `json:"password"` //The password of the hook + HookId int64 `json:"hook_id"` //Hook ID + HookUrl string `json:"hook_url"` //Hook routing + Timestamp string + Sign string //The signature calculated by the hook based on the key + Comment *HookComment `json:"comment"` //Comment data + Note string //Comment data of the commented target + NoteableType string `json:"noteable_type"` //Target type being reviewed + NoteableId int64 `json:"noteable_id"` //ID of the target being reviewed + Title string //Target title being commented + PerId string `json:"per_iid"` //Target ID being reviewed + ShortCommitId string `json:"short_commit_id"` //The short sha in the commented commit submission + Issue *HookIssue +} + +type IssuePayload struct { + HookId int64 `json:"hook_id"` // Hook id. + HookUrl string `json:"hook_url"` // route + HookName string `json:"hook_name"` // issue_hooks。 + Password string `json:"password"` // Hook code + Action string //issue status + Issue HookIssue //issue information + Sender HookUser //The user information that triggered the hook. + TargetUser HookUser `json:"target_user"` //User information entrusted to handle the issue + User HookUser //issue creator + Assignee HookUser //issue principal + Iid string //issue Logo + Title string //issue title + Description string //issue description + State string //issue status + Url string //issue URL on code cloud + Repository HookIssueRepository +} + +type PullRequest struct { + Id int64 + Number int + State string + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt time.Time `json:"closed_at"` + MergedAt time.Time `json:"merged_at"` +} + +type PullRequestIssue struct { + Id int64 + Number string + Repo string + CveNumber string + Branch string + BrFlag bool +} + +//GetLabelsStr labels slice to string +func (ih *HookIssue) GetLabelsStr() (labels string) { + var lb []string + for _, v := range ih.Labels { + lb = append(lb, v.Name) + } + return strings.Join(lb, ",") +} + +//ReplaceLabelToStr delete old label and add new label and return label string +func (ih HookIssue) ReplaceLabelToStr(oLabel, nLabel string) (labels string) { + var lb []string + es := false + for _, v := range ih.Labels { + if v.Name == nLabel { + es = true + } + if v.Name != oLabel { + lb = append(lb, v.Name) + } + } + if !es { + lb = append(lb, nLabel) + } + return strings.Join(lb, ",") +} diff --git a/cve-vulner-manager/models/initdb.go b/cve-vulner-manager/models/initdb.go new file mode 100644 index 0000000000000000000000000000000000000000..b5a071f31c4f046a2d69e6b7c8633d7e944814e1 --- /dev/null +++ b/cve-vulner-manager/models/initdb.go @@ -0,0 +1,83 @@ +package models + +import ( + "cvevulner/common" + "database/sql" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + _ "github.com/go-sql-driver/mysql" +) + +//InitDb init database +func Initdb() bool { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return false + } + //ConnDb() + dbhost := BConfig.String("mysql::dbhost") + dbport := BConfig.String("mysql::dbport") + dbuser := BConfig.String("mysql::dbuser") + dbname := BConfig.String("mysql::dbname") + dbpwd := BConfig.String("mysql::dbpwd") + //key := beego.AppConfig.String("key") + //key1 := []byte(key) + //bytes, _ := common.DePwdCode(dbpwd, key1) + maxidle, lerr := BConfig.Int("mysql::maxidle") + if lerr != nil { + maxidle = 30 + } + + maxconn, lerr := BConfig.Int("mysql::maxconn") + if lerr != nil { + maxconn = 3000 + } + //dns := dbuser + ":" + string(bytes) + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" + dns := dbuser + ":" + dbpwd + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" + errx := orm.RegisterDriver("mysql", orm.DRMySQL) + if errx != nil { + logs.Error("RegisterDriver, orm err: ", errx) + return false + } + errorm := orm.RegisterDataBase("default", "mysql", dns, maxidle, maxconn) + if errorm != nil { + logs.Error("RegisterDataBase failed", "errorm: ", errorm) + return false + } + logs.Info("Initdb, mysql connection is successful") + res := CreateDb() + if res { + logs.Info("mysql table init success!") + } else { + logs.Error("mysql table init failed!") + return false + } + return true +} + +func ConnDb() (*sql.DB, error) { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("ConnDb, config init error:", err) + return nil, err + } + dbhost := BConfig.String("mysql::dbhost") + dbport := BConfig.String("mysql::dbport") + dbuser := BConfig.String("mysql::dbuser") + dbname := BConfig.String("mysql::dbname") + dbpwd := BConfig.String("mysql::dbpwd") + key := beego.AppConfig.String("key") + key1 := []byte(key) + bytes, _ := common.DePwdCode(dbpwd, key1) + dns := dbuser + ":" + string(bytes) + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" + db, err := sql.Open("mysql", dns) + if err != nil { + logs.Error("Error connecting to the database: ", err, "@tcp("+dbhost+":"+dbport+")/"+dbname) + return nil, err + } + logs.Info("ConnDb, mysql connection is successful!") + return db, nil +} diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go new file mode 100644 index 0000000000000000000000000000000000000000..f2c29dd5f143fb5ae9bd4b642fbc08b1a2267dd8 --- /dev/null +++ b/cve-vulner-manager/models/issue.go @@ -0,0 +1,731 @@ +package models + +import ( + "cvevulner/common" + "cvevulner/util" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "strings" + "sync" +) + +var mutex sync.Mutex + +//QueryIssueCveByNum query issue by cve num +func QueryIssueCveByNum(cvenum, repoPath string, organizationID int8) (GiteOriginIssue, bool) { + o := orm.NewOrm() + var cve GiteOriginIssue + err := o.Raw("select * from cve_gite_origin_issue where "+ + "cve_number = ? and repo_path = ? and organizate_id = ?", + cvenum, repoPath, organizationID).QueryRow(&cve) + if err != nil { + logs.Info("QueryIssueCveByNum, cve_gite_origin_issue, Can't find,"+ + " it's not an error, you need to add data, ", cvenum, repoPath) + return cve, false + } else { + return cve, true + } +} + +//QueryIssue query issuewhitelist +func QueryIssueWhitelist(status int8) ([]IssueRepoWhitelist, error) { + o := orm.NewOrm() + var irw []IssueRepoWhitelist + if status == 0 { + num, err := o.Raw("select repo_id,package_name,version,branchs,status,create_time," + + "update_time,delete_time" + + " from cve_issue_repo_whitelist order by repo_id desc").QueryRows(&irw) + if err == nil && num > 0 { + logs.Info("QueryIssueWhitelist, search result: ", irw) + } else { + logs.Info("QueryIssueWhitelist, The current issue whitelist is empty, cur_time:", + common.GetCurTime(), ",err: ", err) + } + return irw, err + } else { + num, err := o.Raw("select repo_id,package_name,version,branchs,status,create_time,"+ + "update_time,delete_time"+ + " from cve_issue_repo_whitelist where status = ? ", status).QueryRows(&irw) + if err == nil && num > 0 { + logs.Info("QueryIssueWhitelist, search result: ", irw) + } else { + logs.Info("QueryIssueWhitelist, The current issue whitelist is empty, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return irw, err + } +} + +//QueryIssue query center +func QueryIssueByPackName(packName, days string, prcnum int) ([]VulnCenter, error) { + o := orm.NewOrm() + var vc []VulnCenter + num, err := o.Raw("select * from cve_vuln_center where pack_name = ? and "+ + "update_time >= ? and cve_status in (?, ?) "+ + "order by cve_id asc limit ?", packName, days, 0, 1, prcnum).QueryRows(&vc) + if err == nil && num > 0 { + logs.Info("QueryIssueByPackName, cve_vuln_center , search result：", vc, "Query conditions： packName: ", packName) + } else { + logs.Info("QueryIssueByPackName, cve_vuln_center, There is currently no cve, need to submit an issue, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return vc, err +} + +//QueryIssue query center +func QueryIssue(days string, prcnum int) ([]VulnCenter, error) { + o := orm.NewOrm() + var vc []VulnCenter + num, err := o.Raw("select * from cve_vuln_center where cve_status in (?, ?) "+ + "order by cve_id asc limit ?", 0, 1, prcnum).QueryRows(&vc) + if err == nil && num > 0 { + logs.Info("QueryIssue, cve_vuln_center, search result: ", vc) + } else { + logs.Info("QueryIssue, cve_vuln_center, There is currently no cve, need to submit an issue, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return vc, err +} + +func GetVulnCenterByCVEID(vc *VulnCenter, cveId int64, fields ...string) error { + o := orm.NewOrm() + var fieldsStr string + if len(fields) == 0 { + fieldsStr = "*" + } else { + fieldsStr = strings.Join(fields, ",") + } + sqlStr := fmt.Sprintf(`select %v from cve_vuln_center where cve_id=?`, fieldsStr) + err := o.Raw(sqlStr, cveId).QueryRow(vc) + return err +} + +//QueryIssueSecurity 查找安全公告 +func QueryIssueSecurity(cveId int64, cveNum string) (SecurityNotice, error) { + o := orm.NewOrm() + var sn SecurityNotice + err := o.Raw("select sec_id, cve_id,cve_num,openeuler_id,introduction,summary,theme,"+ + "description,influence_component,affect_product,reference_link"+ + " from cve_security_notice where cve_id = ? and cve_num = ?", cveId, cveNum).QueryRow(&sn) + if err == nil { + logs.Info("QueryIssueSecurity, cve_security_notice, search result:", sn) + } else { + logs.Info("QueryIssueSecurity, cve_security_notice, cveId: ", cveId, ", err: ", err) + } + return sn, err +} + +func QueryIssueScore(cveId int64) (Score, error) { + o := orm.NewOrm() + var sc Score + err := o.Raw("select *"+ + " from cve_score where cve_id = ?", cveId).QueryRow(&sc) + if err == nil { + logs.Info("QueryIssueScore, cve_score, search result: ", sc) + } else { + logs.Info("QueryIssueScore, cve_score, cveId: ", cveId, ", err: ", err) + } + return sc, err +} + +func GetIssueTemplateByStatus(status int8, templateId int64, num int) (issueTemp []IssueTemplate, err error) { + sql := `select * from cve_issue_template where status = %d and template_id > %d order by template_id limit %d` + sql = fmt.Sprintf(sql, status, templateId, num) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemp) + return +} + +func GetIssueTemplateByColName(it *IssueTemplate, colName ...string) error { + o := orm.NewOrm() + err := o.Read(it, colName...) + return err +} + +func GetSpecIssueAssignee(sia *SpecIssueAssigness, colName ...string) error { + o := orm.NewOrm() + err := o.Read(sia, colName...) + return err +} + +func UpdateIssueTemplate(it *IssueTemplate, fields ...string) error { + logs.Info("Update===>", it, fields) + o := orm.NewOrm() + _, err := o.Update(it, fields...) + return err +} + +// query data +func QueryIssueTemplateAssociation(ita *IssueTemplateAssociation, colName ...string) error { + o := orm.NewOrm() + err := o.Read(ita, colName...) + return err +} + +// update data +func UpdateIssueTemplateAssociation(ita *IssueTemplateAssociation, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ita, fields...) + return err +} + +func InsertIssueTemplateAssociation(ita *IssueTemplateAssociation) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ita) + return id, err +} + +func UpdateScore(s *Score, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(s, fields...) + return err +} + +func UpdatePackageByCveId(pkgStr string, cveId int64) error { + mutex.Lock() + defer mutex.Unlock() + if pkgStr == "" || cveId == 0 { + return errors.New("param pkgStr,cveId must be not empty") + } + ps := strings.Split(pkgStr, ",") + if len(ps) > 0 { + sec := struct { + SecId int64 + }{} + secSql := `SELECT sec_id FROM cve_security_notice WHERE cve_id = ?` + o := orm.NewOrm() + err := o.Raw(secSql, cveId).QueryRow(&sec) + if err != nil { + return err + } + for _, v := range ps { + tv := util.TrimString(v) + if tv == "" { + continue + } + pkg := Package{SecId: sec.SecId, PackName: tv} + errx := o.Read(&pkg, "sec_id", "pack_name") + if errx == orm.ErrNoRows { + pkg.PackUrl = fmt.Sprintf(`https://repo.openeuler.org/openEuler-20.03-LTS/update/aarch64/Packages/%s`, tv) + _, errx = o.Insert(&pkg) + } + } + return nil + } + return nil +} + +func ReplacePackageByCveId(pkgList []string, cveId int64, affectBranch string) error { + sec := struct { + SecId int64 + }{} + secSql := `SELECT sec_id FROM cve_security_notice WHERE cve_id = ?` + o := orm.NewOrm() + err := o.Raw(secSql, cveId).QueryRow(&sec) + if err != nil { + return err + } + delPkgSql := `DELETE FROM cve_package WHERE sec_id = ? and affect_product = ?` + _, err = o.Raw(delPkgSql, sec.SecId, affectBranch).Exec() + if err != nil { + logs.Error("ReplacePackageByCveId, delete cve_package error:", err) + } + pkgData := make([]Package, 0) + for _, v := range pkgList { + if strings.Trim(v, " ") == "" { + continue + } + platform := "source" + if strings.Contains(v, ".x86_64.") { + platform = "x86_64" + } else if strings.Contains(v, ".aarch64.") || strings.Contains(v, ".noarch.") { + platform = "aarch64" + } + pkgUrl := fmt.Sprintf(`https://repo.openeuler.org/%s/update/%s/Packages/%s`, affectBranch, platform, v) + pv := Package{SecId: sec.SecId, PackName: v, PackUrl: pkgUrl, AffectProduct: affectBranch, Source: platform} + pkgData = append(pkgData, pv) + } + _, err = o.InsertMulti(1, pkgData) + return err +} + +func QueryPackageByCveId(cveId int64) ([]Package, error) { + sqlStr := `SELECT * FROM cve_package WHERE sec_id = (SELECT sec_id FROM cve_security_notice WHERE cve_id = ?)` + var res []Package + o := orm.NewOrm() + _, err := o.Raw(sqlStr, cveId).QueryRows(&res) + return res, err +} + +func InsertIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { + o := orm.NewOrm() + if issTempId, err = o.Insert(it); err == nil && issTempId > 0 { + logs.Info("InsertIssueTemplate, insert cve_issue_template success, issTempId: ", + issTempId, "cveNum: ", it.CveNum) + } else { + logs.Error("InsertIssueTemplate, insert cve_issue_template failed, cveNum:", + it.CveNum, ", err: ", err) + return 0, err + } + return issTempId, nil +} + +func UpdateIssueTemplateAll(it *IssueTemplate) (issTempId int64, err error) { + o := orm.NewOrm() + if num, err := o.Update(it); err == nil { + logs.Info("UpdateIssueTemplateAll, update cve_issue_template success, num: ", num, "cveNum: ", it.CveNum) + } else { + logs.Error("UpdateIssueTemplateAll, update cve_issue_template failed, cveNum:", + it.CveNum, ", err: ", err) + return 0, err + } + return it.TemplateId, nil +} + +func DeleteIssueTemplate(issTempId int64) error { + o := orm.NewOrm() + var it = IssueTemplate{TemplateId: issTempId} + id, dErr := o.Delete(&it) + if dErr == nil && id > 0 { + logs.Info("DeleteIssueTemplate, delete cve_issue_template success, issTempId: ", issTempId) + } else { + logs.Error("DeleteIssueTemplate, delete cve_issue_template failed, issTempId: ", + issTempId, ", err: ", dErr) + } + return dErr +} + +func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { + o := orm.NewOrm() + if it.TemplateId == 0 { + var issTempId int64 + if issTempId, err = o.Insert(it); err == nil { + logs.Info("CreateIssueTemplate, insert cve_issue_template success, "+ + "issTempId: ", issTempId, ", cveNum: ", it.CveNum) + } else { + logs.Error("CreateIssueTemplate, insert cve_issue_template failed, cveNum:", + it.CveNum, ", err: ", err) + return 0, err + } + return issTempId, nil + } else { + if num, err := o.Update(it); err == nil { + logs.Info("CreateIssueTemplate, update cve_issue_template success, "+ + "num: ", num, ", cveNum: ", it.CveNum) + } else { + logs.Error("CreateIssueTemplate, update cve_issue_template failed, "+ + "cveNum:", it.CveNum, ", err: ", err) + return 0, err + } + return it.TemplateId, nil + } +} + +func GetAllIssueHook(hookId int64, count, status int) (localh []IssueHooks, value bool) { + o := orm.NewOrm() + var localIh []IssueHooks + var num int64 + num, err := o.Raw("select *"+ + " from cve_issue_hooks where status = ? and id > ? order by id limit ?", + status, hookId, count).QueryRows(&localIh) + if err == nil && num > 0 { + return localIh, true + } else { + logs.Info("GetAllIssueHook, cve_issue_hooks, err: ", err) + return localIh, false + } +} + +func UpdateIssueHook(id int64, status int8) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_issue_hooks SET status = ? WHERE id = ?", status, id).QueryRow() + return +} + +func GetIssueHook(ih *IssueHooks) (localh []IssueHooks, value bool) { + o := orm.NewOrm() + var localIh []IssueHooks + var num int64 + num, err := o.Raw("select *"+ + " from cve_issue_hooks where owner = ? and repo = ? and status = ?", + ih.Owner, ih.Repo, ih.Status).QueryRows(&localIh) + if err == nil && num > 0 { + return localIh, true + } else { + logs.Info("GetIssueHook, cve_issue_hooks, "+ + "cveId: ", ih.CveId, ", err: ", err) + return localIh, false + } +} + +func CreateDepositHooks(ih *IssueHooks) (issHookId int64, err error) { + o := orm.NewOrm() + var localIh IssueHooks + localIh.CveId = ih.CveId + localIh.IssueNum = ih.IssueNum + errx := o.Raw("select *"+ + " from cve_issue_hooks where owner = ? and repo = ? and status = ?", + ih.Owner, ih.Repo, ih.Status).QueryRow(&localIh) + if errx == nil { + ih.Id = localIh.Id + if num, err := o.Update(ih); err == nil { + logs.Info("CreateDepositHooks, update cve_issue_hook success, "+ + "issHookId: ", num, "IssueNum: ", ih.IssueNum) + } else { + logs.Error("CreateDepositHooks, update cve_issue_hook failed, "+ + "IssueNum:", ih.IssueNum, ", err: ", err) + return 0, err + } + return ih.Id, nil + } else { + var issHookId int64 + if issHookId, err = o.Insert(ih); err == nil { + logs.Info("CreateDepositHooks, insert cve_issue_hook success, "+ + "issHookId: ", issHookId, ", IssueNum: ", ih.IssueNum) + } else { + logs.Error("CreateDepositHooks, insert cve_issue_hook failed, "+ + "IssueNum:", ih.IssueNum, ", err: ", err) + return 0, err + } + return issHookId, nil + } +} + +func UpdateSecNotice(sec *SecurityNotice) (secId int64, err error) { + o := orm.NewOrm() + var localSec SecurityNotice + localSec.CveId = sec.CveId + localSec.CveNum = sec.CveNum + errx := o.Raw("select *"+ + " from cve_security_notice where cve_id = ? and cve_num = ?", + sec.CveId, sec.CveNum).QueryRow(&localSec) + if errx == nil { + sec.SecId = localSec.SecId + sec.OpenId = localSec.OpenId + sec.InfluenceComponent = localSec.InfluenceComponent + sec.Description = localSec.Description + sec.Summary = localSec.Summary + sec.AffectStatus = localSec.AffectStatus + sec.Status = localSec.Status + sec.ReferenceLink = localSec.ReferenceLink + if num, err := o.Update(sec); err == nil { + logs.Info("UpdateSecNotice, update cve_security_notice success, "+ + "SecID: ", num, ",CveNum: ", sec.CveNum) + } else { + logs.Error("UpdateSecNotice, update cve_security_notice failed, "+ + "SecID:", sec.SecId, ", err: ", err) + return 0, err + } + return sec.SecId, nil + } else { + var SecId int64 + if SecId, err = o.Insert(sec); err == nil { + logs.Info("UpdateSecNotice, insert cve_security_notice success, "+ + "SecID: ", SecId, ", CveNum: ", sec.CveNum) + } else { + logs.Error("UpdateSecNotice, insert cve_security_notice failed, "+ + "CveNum:", sec.CveNum, ", err: ", err) + return 0, err + } + return SecId, nil + } +} + +func UpdateIssueStatus(iss VulnCenter, status int8) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_vuln_center SET cve_status = ? WHERE cve_id = ? and cve_num = ?", + status, iss.CveId, iss.CveNum).QueryRow() + return +} + +func UpdateLockIssueStatus(beStatus, afStatus int8) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_vuln_center SET cve_status = ? WHERE cve_status = ?", + afStatus, beStatus).QueryRow() + return +} + +func GetIssueSpecErrByColName(se *SpecError, colName ...string) error { + o := orm.NewOrm() + err := o.Read(se, colName...) + return err +} + +func GetOrgCveDesc(ord *OriginUpstreamDesc, colName ...string) error { + o := orm.NewOrm() + err := o.Read(ord, colName...) + return err +} + +func LockUpdateIssueStatus(cveId int64, cveNum string, status int8) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_vuln_center SET cve_status = ? "+ + "WHERE cve_id = ? and cve_num = ? and cve_status in (0,1)", + status, cveId, cveNum).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("LockUpdateIssueStatus, update cve_vuln_center row affected "+ + "nums: ", num, ",cveId: ", cveId) + return true + } + return false + } else { + logs.Error("LockUpdateIssueStatus, update cve_vuln_center failed, "+ + "cveId: ", cveId, ", err: ", err) + return false + } +} + +func UnLockUpdateIssueStatus(beforeStatus, afterStatus int8, updateTime, beforeTime string) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_vuln_center SET cve_status = ?,update_time = ? WHERE cve_status = ? and update_time < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateIssueStatus, update cve_vuln_center row affected nums: ", num) + return true + } + return false + } else { + logs.Error("UnLockUpdateIssueStatus, update cve_vuln_center failed, cve_vuln_center, err: ", err) + return false + } +} + +func UpdateIssueScore(iss VulnCenter, status int8) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_score SET n_score_status = ? WHERE cve_id = ? and cve_num = ?", + status, iss.CveId, iss.CveNum).QueryRow() + return +} + +func UpdateIssueScoreRe(iss VulnCenter, status int8) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_score_record SET status = ? WHERE cve_id = ? and status = ?", + status, iss.CveId, 0).QueryRow() + return +} + +func UpdateIssueCommentId(issueNum, cveNum string, commentId int64) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_issue_template SET comment_id = ? WHERE issue_num = ? and cve_num = ?", + commentId, issueNum, cveNum).QueryRow() + return +} + +//IssueExistByNumber Determine whether the issue exists +func IssueExistByNumber(number string, id int64) bool { + if number == "" { + return false + } + o := orm.NewOrm() + exit := o.QueryTable("cve_issue_template").Filter("issue_num", number).Filter("issue_id", id).Exist() + return exit +} + +func GetIssueTplByCveNum(num string) (list []IssueTemplate, err error) { + o := orm.NewOrm() + _, err = o.QueryTable("cve_issue_template").Filter("cve_num", num).All(&list) + return +} + +func CreateIssueRecord(icr *IssueCreateRecord) (issueRecordId int64, err error) { + o := orm.NewOrm() + var localIcr IssueCreateRecord + errx := o.Raw("select *"+ + " from cve_issue_create_record where cve_id = ? and cve_num = ? and cve_version = ?", + icr.CveId, icr.CveNum, icr.CveVersion).QueryRow(&localIcr) + if errx != nil || localIcr.IssueRecordId == 0 { + var issueRecordId int64 + if issueRecordId, err = o.Insert(icr); err == nil { + logs.Info("CreateIssueRecord, insert cve_issue_create_record success, "+ + "issueRecordId: ", issueRecordId, ", cveNum: ", icr.CveNum) + } else { + logs.Error("CreateIssueRecord, insert cve_issue_create_record failed, "+ + "cveData:", icr, ", err: ", err) + return 0, err + } + return issueRecordId, nil + } else { + icr.IssueRecordId = localIcr.IssueRecordId + if num, err := o.Update(icr); err == nil { + logs.Info("CreateIssueRecord, update cve_issue_create_record success,"+ + " num: ", num, ", cveNum: ", icr.CveNum) + } else { + logs.Error("CreateIssueRecord, update cve_issue_create_record failed,"+ + " cveData:", icr, ", err: ", err) + return 0, err + } + return icr.IssueRecordId, nil + } +} + +//QueryIssue query center +func QueryIssueTemplateByTime(beforeTime string, prcnum int, + templateId int64, status int8) ([]IssueTemplate, error) { + o := orm.NewOrm() + var it []IssueTemplate + num, err := o.Raw("select *"+ + " from cve_issue_template where create_time >= ? and status < ? and template_id > ? "+ + "order by template_id asc limit ?", beforeTime, status, templateId, prcnum).QueryRows(&it) + if err == nil && num > 0 { + return it, nil + } + return it, err +} + +func QueryIssueTemplateByLink(beforeTime string, prcnum int, + templateId int64, status int8) ([]IssueTemplate, error) { + o := orm.NewOrm() + var it []IssueTemplate + num, err := o.Raw("select *"+ + " from cve_issue_template where sec_link = '' and status = ? "+ + "and template_id > ? "+ + "order by template_id asc limit ?", status, templateId, prcnum).QueryRows(&it) + if err == nil && num > 0 { + return it, nil + } + return it, err +} + +func QueryAllCveVuln(prcnum int, cveId int64) ([]VulnCenter, error) { + o := orm.NewOrm() + var vc []VulnCenter + num, err := o.Raw("select *"+ + " from cve_vuln_center where cve_id > ? "+ + "order by cve_id asc limit ?", cveId, prcnum).QueryRows(&vc) + if err == nil && num > 0 { + return vc, nil + } + return vc, err +} + +func QueryIssueStatisticEmail() ([]IssueStatisticsMailList, error) { + o := orm.NewOrm() + var ism []IssueStatisticsMailList + num, err := o.Raw("SELECT * FROM cve_issue_statistics_mail_list").QueryRows(&ism) + logs.Info("cve_issue_statistics_mail_list: ", num, err) + return ism, err +} + +func QueryLowCredibilityCve(days string, prcnum, + credibilityLevel int, cveId int64) ([]OriginUpstream, int64, error) { + o := orm.NewOrm() + var gs []OriginUpstream + num, err := o.Raw("select cve_id,cve_un_ids,cve_num, update_type,cve_packname,"+ + "git_packname,cve_title,affect_porduct,cnnvd_id,cnvd_id,published_date,"+ + "vul_status,cve_status,version,credibility_level"+ + " from cve_origin_upstream where update_time >= ? and cve_status in (?, ?) "+ + " and credibility_level > ? and cve_id > ? "+ + "order by cve_id asc limit ?", days, 0, 1, credibilityLevel, cveId, prcnum).QueryRows(&gs) + if err == nil && num > 0 { + logs.Info("QueryLowCredibilityCve, cve_origin_upstream, serach result: ", num) + } else { + logs.Info("QueryLowCredibilityCve, There is currently no new or updated cve, cur_time:", + common.GetCurTime(), "err: ", err) + } + return gs, num, err +} + +func UpdateTempleteError(errDesc, cveNum string, cveId int64) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_issue_template SET error_description = ?, update_time = ? WHERE cve_id = ? and cve_num = ?", + errDesc, common.GetCurTime(), cveId, cveNum).QueryRow() + return 0 +} + +func QueryGiteOriginIssue(cveNum, path string, organizationID int8) (gi []GiteOriginIssue) { + o := orm.NewOrm() + _, err := o.Raw("select * from cve_gite_origin_issue where "+ + "cve_number = ? and repo_path = ? and organizate_id = ?", + cveNum, path, organizationID).QueryRows(&gi) + if err != nil { + logs.Error("QueryGiteOriginIssue, err: ", err) + } + return +} + +func QueryEulerOriginBRepo(packName, branch string) (ori []OpenEulerRepoOrigin) { + o := orm.NewOrm() + brandStr := "%" + branch + "%" + sql := fmt.Sprintf(`select * from cve_open_euler_repo_origin where package_name='%s' and branch like '%s'`, packName, brandStr) + logs.Info("sql===> ", sql) + num, err := o.Raw(sql).QueryRows(&ori) + if err == nil && num > 0 { + logs.Info("QueryEulerOriginBRepo, cve_open_euler_repo_origin, search result: ", ori) + } else { + logs.Info("QueryEulerOriginBRepo, cve_open_euler_repo_origin, There is currently no cve, need to submit an issue, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return +} + +func GetCommunityYamlConfigAll() (cyc []CommunityYamlConfig) { + o := orm.NewOrm() + num, getErr := o.Raw("select table_name,organizate_id" + + " from cve_community_yaml_config order by id asc").QueryRows(&cyc) + if num == 0 { + logs.Error("GetCommunityYamlConfigAll, Get data does not exist, getErr: ", getErr) + } + return +} + +func InsertCommunityYamlConfig(cyc *CommunityYamlConfig) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(cyc) + return id, err +} + +type CommunityPackage struct { + Id int64 `orm:"column(id)"` + PackageName string `orm:"column(package_name)" description:"组件名称"` + PackageVersion string `orm:"column(version)" description:"组件版本"` + RepoName string `orm:"column(repo_name)" description:"仓库名称"` +} + +func GetCommunityYamlList(table string, id int64, count int) (cp []CommunityPackage) { + o := orm.NewOrm() + sql := fmt.Sprintf("select id,package_name,version,repo_name from %s "+ + "where id > %d order by id asc limit %d", table, id, count) + num, getErr := o.Raw(sql).QueryRows(&cp) + if num == 0 { + logs.Error("GetCommunityYamlList, Get data does not exist, geErr: ", getErr) + } + return +} + +func QueryOriginUpstreamByName(pkName string, credibilityLevel int) (ous []OriginUpstream) { + o := orm.NewOrm() + packageName := "%" + pkName + "%" + sql := fmt.Sprintf("select * from cve_origin_upstream where "+ + "credibility_level <= %d and git_packname like '%s' ORDER BY cve_id DESC", credibilityLevel, packageName) + //logs.Info("sql: ", sql) + num, getErr := o.Raw(sql).QueryRows(&ous) + if num == 0 { + logs.Error("QueryOriginUpstreamByName, Get data does not exist, geErr: ", getErr) + } + return +} + +func QueryIssueCommunityStatistics(eoi *IssueCommunityStatistics, field ...string) error { + o := orm.NewOrm() + err := o.Read(eoi, field...) + return err +} + +// insert data +func InsertIssueCommunityStatistics(eoi *IssueCommunityStatistics) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(eoi) + return id, err +} + +func UpdateOriginUpstream(eoi *OriginUpstream, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(eoi, fields...) + return err +} diff --git a/cve-vulner-manager/models/login.go b/cve-vulner-manager/models/login.go new file mode 100644 index 0000000000000000000000000000000000000000..62dcd79360008093912dba596c0bf9372527a85d --- /dev/null +++ b/cve-vulner-manager/models/login.go @@ -0,0 +1,30 @@ +package models + +import ( + "github.com/astaxie/beego/orm" + "time" +) + +func GetCveUserByUser(username, pwd string) (maps []orm.Params, err error) { + o := orm.NewOrm() + var num int64 + num, err = o.Raw("select user_id FROM cve_other_user WHERE user_name=? and pass_word=?", + username, pwd).Values(&maps) + if err == nil && num > 0 { + return maps, nil + } + return nil, err +} + +func UpdateToken(Userid interface{}, token string, newTime time.Time) bool { + o := orm.NewOrm() + _ = o.Raw("update cve_other_user set aes_key=?,expiration_time=? where user_id=?", + token, newTime, Userid).QueryRow() + return true +} + +func GetUserByToken(token string, ou *OtherUser) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_other_user WHERE aes_key=?", token).QueryRow(ou) + return err +} diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go new file mode 100644 index 0000000000000000000000000000000000000000..db9eac197a575203594fb09286a177b4475d34c5 --- /dev/null +++ b/cve-vulner-manager/models/modeldb.go @@ -0,0 +1,1048 @@ +package models + +import ( + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "time" +) + +type IpWhite struct { + IPId int `orm:"pk;auto;column(ip_id)"` + MachineName string `orm:"size(128);column(machine_name)" description:"机器名称"` + MachineIp string `orm:"size(128);column(machine_ip);index" description:"机器ip"` + AccessCount int64 `orm:"default(0);column(access_count)" description:"0: 不受限制; 10000:可以连续访问次数"` +} + +type OtherUser struct { + UerId int `orm:"pk;auto;column(user_id)"` + UserName string `orm:"size(64);column(user_name)"` + PassWord string `orm:"size(256);column(pass_word)"` + AesKey string `orm:"size(512);colnum(aes_key)"` + ExpirationTime string `orm:"size(32);column(expiration_time)" description:"token的过期时间"` + CreateTime time.Time `orm:"auto_now;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type AdminUser struct { + UerId int `orm:"pk;auto;column(user_id)"` // 自增的值 + UserName string `orm:"size(64);column(user_name)"` + PassWord string `orm:"size(256);column(pass_word)"` + CreateTime time.Time `orm:"auto_now;type(datetime);column(create_time)"` +} + +type VulnCenter struct { + CveId int64 `orm:"pk;auto;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` + CveLevel string `orm:"size(32);column(cve_level)" description:"nvd 致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求); + 5:issue已经创建过，不符合要求，不处理;6:字段为空(评分,描述等);7:issue已被删除不再创建;8:分支和版本信息不想符合"` + CveVersion string `orm:"size(512);column(cve_version)" description:"cve归属版本,版本合并"` + RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` + FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` + FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称的仓库名称"` + RepoName string `orm:"size(512);column(repo_name)" description:"cve对应得包名称的组件名称"` + CveUrl string `orm:"type(text);column(cve_url)" description:"cve下载链接"` + IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` + DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` + CveDetailUrl string `orm:"type(text);column(cve_detail_url)" description:"cve详情链接"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type OpenEulerSA struct { + OpenId int64 `orm:"pk;auto;column(openeuler_id)"` + CveId int64 `orm:"index;column(cve_id)"` + PublicDate string `orm:"size(16);column(public_date);null" description:"安全公告时间"` + OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告"` +} + +type ScoreRecord struct { + Id int64 `orm:"pk;auto"` + CveId int64 `orm:"index;column(cve_id)"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)"` + NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` + Status int8 `orm:"default(0);column(status)" description:"0:未处理;1:已处理"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` +} + +type Score struct { + Id int64 `orm:"pk;auto"` + CveId int64 `orm:"index;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` + OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` + NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` + OvectorVule string `orm:"size(256);column(o_vector_value)" description:"openeuler vector 原始值"` + NattackVector string `orm:"size(64);column(n_attack_vector)"` + NaccessVector string `orm:"size(256);column(n_access_vector);null" description:"nvd 2.0 评分向量"` + OattackVector string `orm:"size(64);column(o_attack_vector)"` + OaccessVector string `orm:"size(256);column(o_access_vector);null" description:"nvd 2.0 评分向量"` + NattackComplexity string `orm:"size(64);column(n_attack_complexity)"` + NaccessComplexity string `orm:"size(64);column(n_access_complexity);null"` + OattackComplexity string `orm:"size(64);column(o_attack_complexity)"` + OaccessComplexity string `orm:"size(64);column(o_access_complexity);null"` + NprivilegeRequired string `orm:"size(64);column(n_privilege_required)"` + OprivilegeRequired string `orm:"size(64);column(o_privilege_required)"` + NuserInteraction string `orm:"size(64);column(n_user_interaction)"` + OuserInteraction string `orm:"size(64);column(o_user_interaction)"` + Nscope string `orm:"size(64);column(n_scope)"` + Oscope string `orm:"size(64);column(o_scope)"` + Nconfidentiality string `orm:"size(64);column(n_confidentiality)"` + Oconfidentiality string `orm:"size(64);column(o_confidentiality)"` + Nintegrity string `orm:"size(64);column(n_integrity)"` + Ointegrity string `orm:"size(64);column(o_integrity)"` + Navailability string `orm:"size(64);column(n_availability)"` + Oavailability string `orm:"size(64);column(o_availability)"` + Nauthentication string `orm:"size(64);column(n_authentication);null"` + Oauthentication string `orm:"size(64);column(o_authentication);null"` + ScoreType string `orm:"size(16);column(score_type)"` + OpenId int64 `orm:"unique;column(openeuler_id);null"` + Nstatus int8 `orm:"default(0);column(n_score_status)" description:"0:新增评分，无改变；1:修改nvd评分;2:已提交issue；3:已处理"` + Ostatus int8 `orm:"default(0);column(o_score_status)" description:"0:新增评分，无改变；1:修改nvd评分；2：修改评分未通过；3：修改评分已通过"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type SecurityNotice struct { + SecId int64 `orm:"pk;auto;column(sec_id)" description:"安全公告"` + CveId int64 `orm:"index;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` + OpenId int64 `orm:"unique;column(openeuler_id);null" description:"OpenEulerSA"` + Introduction string `orm:"size(256);null;column(introduction)" description:"简介"` + Summary string `orm:"size(256);null;column(summary)" description:"概要"` + Theme string `orm:"type(text);null;column(theme)" description:"主题"` + Description string `orm:"type(text);column(description)" description:"安全公告描述"` + InfluenceComponent string `orm:"size(256);null;column(influence_component)" description:"影响组件"` + AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` + ReferenceLink string `orm:"type(text);null;column(reference_link)" description:"参考链接"` + Status int8 `orm:"default(0);column(sec_status)" description:"0:未发布；1:已发布"` + AffectStatus string `orm:"size(16);null;column(affect_status)" description:"Fixed:已解决；UnFixed:未解决；UnAffected:不影响"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type Package struct { + Id int64 `orm:"pk;auto"` + SecId int64 `orm:"index;column(sec_id)"` + PackName string `orm:"type(text);null;column(pack_name)" description:"包名"` + PackUrl string `orm:"type(text);null;column(pack_url)" description:"包下载链接"` + AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` + Source string `orm:"size(128);null;column(source)" description:"数据源"` +} + +type IssueTemplate struct { + TemplateId int64 `orm:"pk;auto;column(template_id)"` + CveId int64 `orm:"index;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` + OwnedComponent string `orm:"size(512);column(owned_component)" description:"漏洞归属组件"` + OwnedVersion string `orm:"size(512);column(owned_version)" description:"漏洞归属版本"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd评分"` + OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` + NVDVector string `orm:"size(256);column(nvd_vector)" description:"nvd评分向量"` + OpenEulerVector string `orm:"size(256);column(openeuler_vector)" description:"openeuler评分向量"` + CveBrief string `orm:"type(text);column(cve_brief)" description:"漏洞简述"` + CveAnalysis string `orm:"type(text);column(cve_analysis)" description:"影响性分析说明"` + PrincipleAnalysis string `orm:"type(text);column(principle_analysis)" description:"原理分析"` + AffectedVersion string `orm:"size(256);column(affected_version)" description:"受影响的版本"` + Solution string `orm:"type(text);column(solution)" description:"规避方案或消减措施"` + IssueId int64 `orm:"column(issue_id)" description:"issue的id"` + IssueNum string `orm:"size(64);column(issue_num)" description:"issue编号"` + Assignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"` + Status int8 `orm:"default(1);column(status)" description:"1:待办的；2：进行中；3：已完成；4：已拒绝；5： 已挂起; 6:已删除"` + StatusName string `orm:"size(128);column(status_name)" description:"issue状态名称"` + IssueStatus int8 `orm:"default(1);column(issue_status)" description:"1:待分析；2：已正常关闭；3：已分析，待修复；4：已修复；5：已发布；6：已异常关闭"` + IssueLabel string `orm:"size(256);column(issue_label)" description:"issue标签, CVE/FIXED, CVE/UNFIXED"` + MtAuditFlag int8 `orm:"default(0);column(mt_audit_flag)" description:"maintainer 审核： 0：审核中；1：审核通过；2：审核拒绝"` + SaAuditFlag int8 `orm:"default(0);column(sa_audit_flag)" description:"pr审核： 0：审核中；1：审核通过；2：审核拒绝"` + OpAuditFlag int8 `orm:"default(0);column(op_audit_flag)" description:"评分审核： 0：审核中；1：审核通过；2：审核拒绝"` + Owner string `orm:"size(128);column(owner)" description:"仓库地址"` + Repo string `orm:"size(512);column(repo)" description:"仓库路径"` + Title string `orm:"size(512);column(title)" description:"issue标题"` + IssueType string `orm:"size(64);column(issue_type)" description:"CVE和安全问题"` + Collaborators string `orm:"size(128);column(collaborators);null" description:"协助者"` + Milestone string `orm:"size(64);column(milestone);null" description:"里程碑序号"` + Program string `orm:"size(64);column(program);null" description:"项目编号"` + SecurityHole int8 `orm:"default(0);column(security_hole)" description:"是否是私有issue"` + CveLevel string `orm:"size(32);column(cve_level)" description:"openEuler 致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + CommentId int64 `orm:"column(comment_id);null" description:"首条评论id"` + ErrorDescription string `orm:"size(64);column(error_description);null" description:"cve错误信息,cve与当前仓库错误信息保存"` + SecLink string `orm:"size(512);column(sec_link)" description:"sa安全链接"` + AbiVersion string `orm:"size(256);column(abi_version)" description:"修复是否涉及abi变化（是/否）"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type IssueTemplateAssociation struct { + AssId int64 `orm:"pk;auto;column(ass_id)"` + TemplateId int64 `orm:"unique;column(template_id)"` + SaReleaseTime string `orm:"size(32);column(sa_release_time);null"` + RpmReleaseTime string `orm:"size(32);column(rpm_release_time);null"` +} + +type IssueHooks struct { + Id int64 `orm:"pk;auto"` + CveId int64 `orm:"index;column(cve_id)"` + IssueId int64 `orm:"column(issue_id)" description:"issue的id"` + IssueNum string `orm:"size(64);column(issue_num);index" description:"issue编号"` + HookId int64 `orm:"column(hook_id)" decrtiption:"钩子id"` + Owner string `orm:"size(128);column(owner)" description:"仓库地址"` + Repo string `orm:"size(128);column(repo)" description:"仓库路径"` + HookUrl string `orm:"size(512);column(hook_url)" description:"回调地址"` + PushEvent int8 `orm:"column(push_event);default(1)" description:"Push代码到仓库; 默认1：已监听; 0: 未监听"` + TagPushEvent int8 `orm:"column(tag_push_events);default(1)" description:"提交Tag到仓库, 默认1：已监听; 0: 未监听"` + IssueEvent int8 `orm:"column(issues_events);default(1)" description:"创建/关闭Issue, 默认1：已监听; 0: 未监听"` + NoteEvent int8 `orm:"column(note_events);default(1)" description:"评论了Issue/代码等等, 默认1：已监听; 0: 未监听"` + MergeRequestEvent int8 `orm:"column(merge_requests_events);default(1)" description:"合并请求和合并后, 默认1：已监听; 0: 未监听"` + Status int8 `orm:"default(1);column(status)" description:"1:有效; 2:无效"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type GitPackageTable struct { + TableId int64 `orm:"pk;auto;column(table_id)"` + TableName string `orm:"size(128);column(table_name)" description:"数据库pkginfo下的表名，如：mainline, bringInRely"` + Brands string `orm:"size(256);column(brands)" description:"对应分支信息"` +} + +type GitOpenEuler struct { + GitId int64 `orm:"pk;auto;column(git_id)"` + PackageId int64 `orm:"index;column(package_id)" description:"获取到的包id"` + PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + Release string `orm:"size(128);column(releases)" description:"releases 版本号"` + OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` + License string `orm:"type(text);column(license)" description:"license"` + Feature string `orm:"size(128);column(feature)" description:"特性"` + MainTainer string `orm:"size(128);column(main_tainer)" description:"维护人"` + MainTainLevel int8 `orm:"column(main_tain_level);default(1)" description:"软件包维护级别"` + ReleaseTime string `orm:"size(32);column(release_time)" description:"当前版本发布时间"` + UsedTime string `orm:"size(32);column(used_time)" description:"当天减去所用版本的发布日期"` + LatestVersion string `orm:"size(128);column(latest_version)" description:"最新版本号"` + LatestVersionTime string `orm:"size(32);column(latest_version_time)" description:"最新版本发布时间"` + IssueCount int64 `orm:"column(issue_count);default(0)" description:"该软件包仓库下的issue总数"` + TableId int64 `orm:"index;column(table_id)" description:"表id, 外键"` + TableName string `orm:"size(128);column(table_name)" description:"数据库pkginfo下的表名，如：mainline, bringInRely"` + CpePackName string `orm:"size(512);column(cpe_packname);null" description:"openEuler对应的cpe包名称(或者影响的包名)"` + Status int8 `orm:"default(0);column(status)" description:"0: 代表新数据; 1:已推送; 2:代表已废弃"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type GitOpenEulerTableRelate struct { + RelateId int64 `orm:"pk;auto;column(relate_id)"` + GitId int64 `orm:"index;column(git_id)" description:"GitOpenEuler，外键"` + Brands string `orm:"size(256);column(brands)" description:"对应分支信息"` + TableName string `orm:"size(256);column(table_name)" description:"数据库pkginfo下的表名，如：mainline, bringInRely"` +} + +type GitPackageInfo struct { + DetailId int64 `orm:"pk;auto;column(detail_id)"` + GitId int64 `orm:"index;column(git_id)" description:"包id，外键"` + Ids int64 `orm:"index;column(git_ids)" description:"原始数据id来源"` + PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + Release string `orm:"size(128);column(release)" description:"release 版本号"` + OriginUrl string `orm:"size(512);column(origin_url)" description:"上游社区链接"` + License string `orm:"type(text);column(license)" description:"license"` + Feature string `orm:"size(128);column(feature)" description:"特性"` + MainTainer string `orm:"size(128);column(main_tainer)" description:"维护人"` + MainTainLevel int8 `orm:"column(main_tain_level);default(1)" description:"软件包维护级别"` + GitUrl string `orm:"size(512);column(git_url)" description:"gitee上的地址"` + Summary string `orm:"type(text);column(summary)" description:"summary"` + Decription string `orm:"type(text);column(decription)" description:"decription"` + BuildRequired string `orm:"type(text);column(build_require)" description:"源码包的编译依赖"` + Status int8 `orm:"default(0);column(status)" description:"0: 代表新数据; 1:已推送; 2:代表已废弃"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` +} + +type GitSubPack struct { + SubId int64 `orm:"pk;auto;column(sub_id)"` + DetailId int64 `orm:"index;column(detail_id)" description:"包id，外键"` + Ids int64 `orm:"index;column(ids)" description:"原始数据id来源"` + SubPackName string `orm:"size(256);column(sub_pack_name)" description:"二进制包名"` +} + +type GitSubPackRequire struct { + RequireId int64 `orm:"pk;auto;column(require_id)"` + SubId int64 `orm:"index;column(sub_id)" description:"包id,外键"` + Ids int64 `orm:"index;column(ids)" description:"原始数据id来源"` + RequireName string `orm:"size(256);column(require_name)" description:"二进制包名"` + Providedby string `orm:"type(text);column(providedb)" description:"提供该组件的二进制包"` +} + +type GitSubPackProvides struct { + ProvideId int64 `orm:"pk;auto;column(provide_id)"` + SubId int64 `orm:"index;column(sub_id)" description:"包id，外键"` + Ids int64 `orm:"index;column(ids)" description:"原始数据id来源"` + ProvideName string `orm:"size(256);column(provide_name)" description:"依赖的组件名称"` + Requiredby string `orm:"size(256);column(requiredb)" description:"依赖该组件的二进制包列表"` +} + +type GitSubPackRequiredby struct { + Id int64 `orm:"pk;auto;column(id)"` + ProvideId int64 `orm:"index;column(provide_id)" description:" 包id，外键"` + Requiredby string `orm:"size(256);column(requiredb)" description:"依赖该组件的二进制包列表"` +} + +type OriginUpstream struct { + CveId int64 `orm:"pk;auto;column(cve_id)"` + Ids string `orm:"size(256);column(cve_un_ids);unique" description:" 唯一编号,根据此字段去重数据, 唯一识别码，可以填cve编号"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + UpdateType string `orm:"size(32);column(update_type);" description:"数据上传类型：insert, update, delete"` + CvePackName string `orm:"size(512);column(cve_packname);null" description:"Cve在上游对应的包名"` + PackName string `orm:"size(512);column(git_packname);index;null" description:"Cve对应的openEuler包名称(或者影响的包名)"` + Title string `orm:"type(text);column(cve_title);null" description:"标题"` + AffectProduct string `orm:"size(512);column(affect_porduct);null" description:"Cve影响的组件, 对应"` + CnnvdID string `orm:"size(256);column(cnnvd_id);null" description:"Cnnvd_id"` + CnvdID string `orm:"size(256);column(cnvd_id);null" description:"Cnvd_id"` + PublishedDate string `orm:"size(32);column(published_date);null" description:"漏洞发布日期"` + FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` + FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` + VulStatus string `orm:"size(64);column(vul_status);null" description:"漏洞状态，REJECT, DISPUTED"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` + AffectedScope string `orm:"size(512);column(affected_scope);null" description:"影响范围推理"` + Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` + AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` + IsExit int8 `orm:"default(1);column(is_exit)" description:"1: 当前包对应在src-openEuler有对应仓库; 0: 无;2:临时值"` + Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; + 1:包名通过别名匹配、版本号为漏洞库原始数据; + 2:包名版本号都通过漏洞描述获取; + 3:包名通过漏洞描述获取并通过别名匹配、版本号为漏洞描述获取; + 4:包名版本号通过SA获取、版本号为修复版本号、修复版本以下都视为受影响版本; + 5:包名版本号通过SA获取并通过别名匹配、版本号为修复版本号、修复版本以下都视为受影响版本; + 6:版本号未匹配、包名通过以上任意方式匹配、可信度最低"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OriginUpstreamDesc struct { + DescId int64 `orm:"pk;auto;column(desc_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + EnDescription string `orm:"type(text);column(en_desc);null" description:"cve英文描述"` + ZhDescription string `orm:"type(text);column(zh_desc);null" description:"cve中文描述"` +} + +type OriginUpstreamConfig struct { + ConfId int64 `orm:"pk;auto;column(conf_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Nodes string `orm:"size(32);column(nodes);null" description:"nodes数组"` +} + +type OriginUpstreamConfigNode struct { + NodeId int64 `orm:"pk;auto;column(node_id)"` + ConfId int64 `orm:"index;column(conf_id)" description:"OriginUpstreamConfig 外键"` + Operator string `orm:"size(256);column(operator);null" description:"operator"` +} + +type SecurityReviewer struct { + Id int64 `orm:"pk;auto"` + NameSpace string `orm:"unique" description:"码云空间地址"` + Status int8 `orm:"default(0);column(status)" description:"0: 全部；1:审核人"` +} + +type IssueAssignee struct { + Id int64 `orm:"pk;auto"` + Assignee string `orm:"unique" description:"码云空间地址"` +} + +type OriginUpstreamConfigNodeCpe struct { + CpeId int64 `orm:"pk;auto;column(cpe_id)"` + NodeId int64 `orm:"index;column(node_id)" description:"OriginUpstreamConfigNode 外键"` + Cpe23Uri string `orm:"type(text);column(cpe_uri);null" description:"cpe"` + CpeMatchString string `orm:"type(text);column(cpe_match);null" description:"过滤字符串"` + Vulnerable string `orm:"size(64);column(vulner_able);null" description:"易受攻击"` +} + +type OriginUpstreamImpact struct { + ImpactId int64 `orm:"pk;auto;column(impact_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Impact string `orm:"size(32);column(impact);null" description:"包含v2,v3评分数据"` +} + +type OriginUpstreamImpactScore struct { + ScoreId int64 `orm:"pk;auto;column(score_id)"` + ImpactId int64 `orm:"index;column(impact_id)" description:"OriginUpstreamImpact 外键"` + BaseMetricV3 int8 `orm:"column(base_met_v3);null" description:"v3评分存在为：1; 0:不存在"` + BaseMetricV2 int8 `orm:"column(base_met_v2);null" description:"v2评分存在为：1; 0:不存在"` + CvssV3 int8 `orm:"column(cvss_v3);null" description:"v3评分存在为：1; 0:不存在"` + CvssV2 int8 `orm:"column(cvss_v2);null" description:"v2评分存在为：1; 0:不存在"` + Status int8 `orm:"default(1);column(score_status);null" description:"1:未处理；2：已处理；3：已修改"` +} + +type OriginUpstreamImpactScoreV3 struct { + V3Id int64 `orm:"pk;auto;column(v3_id)"` + ScoreId int64 `orm:"index;column(score_id)" description:"OriginUpstreamImpactScore 外键"` + BaseScore float64 `orm:"digits(10);decimals(1);column(base_score);null" description:"nvd 基准分"` + VectorString string `orm:"size(256);column(vector_value);null" description:"nvd vector 评分向量值"` + AttackComplexity string `orm:"size(64);column(attack_complexity);null" description:"攻击复杂性"` + AttackVector string `orm:"size(64);column(attack_vector);null" description:"攻击目标"` + AvailabilityImpact string `orm:"size(64);column(availability_impact);null" description:"可用性影响"` + BaseSeverity string `orm:"size(64);column(base_severity);null" description:"严重程度"` + UserInteraction string `orm:"size(64);column(user_interaction);null" description:"用户交互"` + PrivilegesRequired string `orm:"size(64);column(privilege_required);null" description:"所需特权"` + Version string `orm:"size(64);column(version);null" description:"Cvss版本"` + ConfidentialityImpact string `orm:"size(64);column(confidentiality_impact);null" description:"可信性影响"` + IntegrityImpact string `orm:"size(64);column(integrity_impact);null" description:"完整性影响"` + Scope string `orm:"size(64);column(scope);null" description:"范围"` + ImpactScore float64 `orm:"digits(10);decimals(1);column(impact_score);null" description:"nvd 影响评分"` + ExploitabilityScore float64 `orm:"digits(10);decimals(1);column(exploitability_score);null" description:"nvd 可利用性评分"` + CveLevel string `orm:"size(32);column(cve_level);null" description:"致命(Critical) >= 9.0;严重(High)>=7.0&&<=8.9;中等(Medium)>4.0 && <=6.9;一般(Low)<=4.0;其他"` +} + +type OriginUpstreamImpactScoreV2 struct { + V2Id int64 `orm:"pk;auto;column(v2_id)"` + ScoreId int64 `orm:"index;column(score_id)" description:"OriginUpstreamImpactScore 外键"` + AcInsufInfo string `orm:"size(64);column(acinsuf_info);null" description:"nvd vector 原始值"` + BaseScore float64 `orm:"digits(10);decimals(1);column(base_score);null" description:"nvd 基准分"` + VectorString string `orm:"size(256);column(vector_value);null" description:"nvd vector 评分向量值"` + AccessComplexity string `orm:"size(64);column(access_complexity);null" description:"攻击复杂性"` + Authentication string `orm:"size(64);column(authentication);null" description:"身份认证"` + AvailabilityImpact string `orm:"size(64);column(availability_impact);null" description:"可用性影响"` + Version string `orm:"size(64);column(version);null" description:"Cvss版本"` + ConfidentialityImpact string `orm:"size(64);column(confidentiality_impact);null" description:"可信性影响"` + IntegrityImpact string `orm:"size(64);column(integrity_impact);null" description:"完整性影响"` + AccessVector string `orm:"size(64);column(access_vector);null" description:"攻击目标"` + UserInteractionRequired string `orm:"size(64);column(user_interaction_required);null" description:"需要用户交互"` + Severity string `orm:"size(64);column(severity);null" description:"严重程度"` + ObtainUserPrivilege string `orm:"size(64);column(obtain_user_privilege);null" description:"获取用户特权"` + ObtainAllPrivilege string `orm:"size(64);column(obtain_all_privilege);null" description:"获取所有特权"` + ObtainOtherPrivilege string `orm:"size(64);column(obtain_other_privilege);null" description:"获取其他特权"` + ImpactScore float64 `orm:"digits(10);decimals(1);column(impact_score);null" description:"nvd 影响评分"` + ExploitabilityScore float64 `orm:"digits(10);decimals(1);column(exploitability_score);null" description:"nvd 可利用性评分"` + CveLevel string `orm:"size(32);column(cve_level);null" description:"致命(Critical) >= 9.0;严重(High)>=7.0&&<=8.9;中等(Medium)>4.0 && <=6.9;一般(Low)<=4.0;其他"` +} + +type OriginUpstreamPoc struct { + PocId int64 `orm:"pk;auto;column(poc_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Source string `orm:"size(512);column(source);null" description:"来源"` + Date string `orm:"size(32);column(date);null" description:"日期"` + Path string `orm:"size(512);column(path);null" description:"文件路径"` + Dbindex string `orm:"size(512);column(dbindex);null" description:"数据库索引"` + Url string `orm:"size(512);column(url);null" description:"下载链接"` + Desc string `orm:"type(text);column(desc);null" description:"说明"` +} + +type OriginUpstreamEvent struct { + EventId int64 `orm:"pk;auto;column(event_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Title string `orm:"size(512);column(title);null" description:"标题"` + Date string `orm:"size(32);column(date);null" description:"日期"` + Url string `orm:"size(512);column(url);null" description:"连接"` + Description string `orm:"type(text);column(description);null" description:"说明"` +} + +type OriginUpstreamReference struct { + RefId int64 `orm:"pk;auto;column(ref_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Name string `orm:"size(512);column(name);null" description:"名称"` + Refsource string `orm:"size(512);column(refsource);null" description:"参考来源"` + Url string `orm:"size(512);column(url);null" description:"链接"` + Tags string `orm:"type(text);column(tags);null" description:"tags"` +} + +type OriginUpstreamVulType struct { + VulId int64 `orm:"pk;auto;column(vul_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Cwe string `orm:"size(256);column(cwe);null" description:"Cwe编号"` + EnDesc string `orm:"type(text);column(en_desc);null" description:"英文描述"` + ZhDesc string `orm:"type(text);column(zh_desc);null" description:"中文描述"` +} + +type OriginUpstreamFixSuggest struct { + FixId int64 `orm:"pk;auto;column(fix_id)"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Detail string `orm:"type(text);column(detail);null" description:"细节"` +} + +type OriginUpstreamFixSuggestRef struct { + FixRefId int64 `orm:"pk;auto;column(fix_ref_id)"` + FixId int64 `orm:"index;column(fix_id)" description:"OriginUpstreamFixSuggest 外键"` + Refsource string `orm:"size(512);column(refsource);null" description:"参考文献，包含以下内容"` + Url string `orm:"type(text);column(url);null" description:"链接"` + Name string `orm:"type(text);column(name);null" description:"名称"` +} + +type OriginUpstreamFixSuggestRefTag struct { + TagId int64 `orm:"pk;auto;column(tag_id)"` + FixRefId int64 `orm:"index;column(fix_ref_id)" description:"OriginUpstreamFixSuggestRef 外键"` + Name string `orm:"size(512);column(name);null" description:"名称"` +} + +//GiteOriginIssue Issues that already exist on Code Cloud +type GiteOriginIssue struct { + Id int64 `orm:"pk;auto"` + IssueId int64 `json:"issue_id" orm:"unique"` + Url string ` description:"issue gitee 链接"` + Number string `orm:"size(50);unique" description:"issue 编号"` + State string `orm:"size(50)" description:"issue 状态"` + Title string `orm:"null" description:"issue 标题"` + CveNumber string `json:"cve_number" description:"CVE 编号"` + Body string `orm:"null;type(text)" description:"issue 主体"` + IssueCreate string `json:"issue_create" description:"issue 创建者"` + IssueAssignee string `json:"issue_assignee" orm:"null" description:"issue 责任人"` + RepoPath string `json:"repo_path" description:"仓库空间地址"` + RepoUrl string `json:"repo_url" description:"仓库码云地址链接"` + IssueType string `json:"issue_type" description:"issue 类型"` + IssueExistTpl bool `json:"issue_exist_tpl" description:"此 issue 是否有对应的模板"` + SecurityHole bool `json:"security_hole" description:"是否为安全漏洞"` + Status int8 `orm:"default(0);column(proc_status)" description:"0:cve新增;1:数据已变化；2：数据已处理；3：数据错误"` + IssueCreateAt time.Time `json:"issue_create_at" orm:"null" description:"issue 创建的时间"` + IssueUpdateAt time.Time `json:"issue_update_at" orm:"null" description:"issue 更新的时间"` + IssueFinishAt time.Time `json:"issue_finish_at" orm:"null" description:"issue 关闭的时间"` + GrabTime time.Time `json:"grab_time" orm:"auto_now;type(datetime)" description:"记录当前issue抓取的时间"` + InfProduct string `json:"inf_product" orm:"null" description:"受影响的产品"` //受影响的产品 + RepoDesc string `json:"repo_desc" orm:"null;size(2048)" description:"cve 对组件的描述"` + IssueState string `orm:"size(50);column(issue_state)" description:"issue 中文状态"` + Owner string `orm:"size(256);column(owner);default(src-openeuler)" description:"issue所属社区"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源src-openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` +} + +type OriginExcel struct { + CveId int64 `orm:"pk;auto;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` + CveUrl string `orm:"type(text);column(cve_url);null" description:"cve下载链接"` + CveVersion string `orm:"size(256);column(cve_version);null" description:"漏洞归属版本"` + PackName string `orm:"size(256);column(pack_name);null" description:"包名"` + ScoreType string `orm:"size(16);column(score_type);null" description:"评分类型：v3"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score);null" description:"nvd评分"` + CveLevel string `orm:"size(32);column(cve_level);null" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + CveDesc string `orm:"type(text);column(cve_desc);null" description:"漏洞简述"` + RepairTime string `orm:"size(32);column(repair_time);null" description:"cve发布时间"` + NVDVector string `orm:"size(256);column(vector_value);null" description:"nvd 评分向量"` + AttackVector string `orm:"size(256);column(attack_vector);null" description:"nvd 3.0评分"` + AccessVector string `orm:"size(256);column(access_vector);null" description:"nvd 2.0 评分"` + AttackComplexity string `orm:"size(64);column(attack_complexity);null"` + AccessComplexity string `orm:"size(64);column(access_complexity);null"` + PrivilegeRequired string `orm:"size(64);column(privilege_required);null"` + UserInteraction string `orm:"size(64);column(user_interaction);null"` + Scope string `orm:"size(64);column(scope);null"` + Confidentiality string `orm:"size(64);column(confidentiality);null"` + Integrity string `orm:"size(64);column(integrity);null"` + Availability string `orm:"size(64);column(availability);null"` + Authentication string `orm:"size(64);column(authentication);null"` + CveStatus int8 `orm:"default(0);column(cve_status);null" description:"0:未处理;1:已修改；2：已处理; 3: 数据错误，不做处理; 4: 版本信息错误; 6: cve已经生成过issue; 7:cve年限不对"` + IsExport int8 `orm:"default(1);column(is_export);null" description:"1: 未导出; 2: 已导出"` + CreateTime string `orm:"type(datetime);column(create_time);null"` + UpdateTime string `orm:"type(datetime);column(update_time);null"` + DeleteTime string `orm:"type(datetime);column(delete_time);null"` +} + +type GitRepoGroups struct { + GroupId int64 `orm:"pk;auto;column(group_id)"` + GroupName string `orm:"index"` +} + +//GiteRepo Code Cloud repo +type GiteRepo struct { + RepoId int64 `orm:"pk;auto;column(repo_id)"` + GroupId int64 `orm:"index;column(group_id)" description:"GitRepoGroups 外键"` + OrgId int64 + OrgPath string `orm:"size(128);column(org_path);null" description:"组织名称"` + Path string `orm:"size(512);column(path);null" description:"仓库名称"` + Status int8 `orm:"default(0);column(status);null" description:"0: 正常;1:已删除"` + Public bool + Private bool + Fork bool + CreateTime string `orm:"type(datetime);column(create_time);null"` + UpdateTime string `orm:"type(datetime);column(update_time);null"` + DeleteTime string `orm:"type(datetime);column(delete_time);null"` +} + +//GiteMember Members of Code Cloud repo +type GiteRepoMember struct { + MemBerId int64 `orm:"pk;auto;column(repo_id)"` + GroupId int64 `orm:"index;column(group_id)" description:"GitRepoGroups 外键"` + MemberName string `orm:"size(512);column(member_name);null" description:"成员名称"` + MemberType string `orm:"size(128);column(member_type);null" description:"Maintainer;Committer"` + CreateTime string `orm:"type(datetime);column(create_time);null"` + UpdateTime string `orm:"type(datetime);column(update_time);null"` + DeleteTime string `orm:"type(datetime);column(delete_time);null"` +} + +//GiteRepoBranch Code Cloud repo Branch +type GiteRepoBranch struct { + Id int64 `orm:"pk;auto"` + RepoId int64 `orm:"index;column(repo_id)" description:"GiteRepo 外键"` + ProtectionUrl string + Name string + Protected string +} + +type ExportRecord struct { + Id int64 `orm:"pk;auto"` + FileName string `orm:"unique"` + FileCode string + State int8 ` description:"0:文件生成中;1 文件可下载；2 文件不可下载"` + CreateTime int64 `orm:"column(create_time)" description:"时间戳"` +} + +type PackageCpe struct { + Id int64 `orm:"pk;auto"` + PackName string `orm:"size(512);column(packname);index;null" description:"Cve对应的openEuler包名称(或者影响的包名)"` + CpePackName string `orm:"size(512);column(cpe_packname);null" description:"openEuler对应的cpe包名称(或者影响的包名)"` + CreateTime string `orm:"size(32);column(create_time);null" description:"createtime"` +} + +type EmailList struct { + Id int64 `orm:"pk;auto"` + EmailName string `orm:"size(512);column(email_name);null" description:"收件人名称"` + EmailType int8 `orm:"index;column(email_type);default(1)" description:"1: excel导出列表"` +} + +type IssueRepoWhitelist struct { + RepoId int64 `orm:"pk;auto;column(repo_id)"` + PackageName string `orm:"column(package_name);size(256);index" description:"包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + Branchs string `orm:"size(512);column(branchs);null" description:"仓库对应的分支"` + Status int8 `orm:"default(1);column(status)" description:"1: 新增; 2:删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type IssueCreateRecord struct { + IssueRecordId int64 `orm:"pk;auto;column(id)"` + CveId int64 `orm:"index;column(cve_id)" description:"VulnCenter 外键"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` + CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + Status int8 `orm:"default(1);column(status)" description:"1:已创建issue；2：未创建issue；3：创建失败; 4: 已导出"` + CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` + RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` + NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` + ErrorDescription string `orm:"type(text);column(error_description);null" description:"创建cve错误信息反馈"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenGussYaml struct { + Id int64 `orm:"pk;auto"` + PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Repo string `orm:"column(repo_name);size(256)" description:"仓名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` + MarkBit int8 `orm:"default(1);column(mark_bit)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SpecIssueAssigness struct { + Id int64 `orm:"pk;auto"` + PackageName string `orm:"column(package_name);size(256);unique" description:"包名称"` + Assignee string `orm:"size(256);column(issue_assignee);" description:"包对应的责任人"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SpecError struct { + Id int64 `orm:"pk;auto;column(id)"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` + Owner string `orm:"size(256);column(cve_owner);index" description:"仓库所在组织"` + Status int8 `orm:"default(1);column(cve_status)" description:"1:正常数据;2:已删除"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type CvrfSaRecord struct { + Id int64 `orm:"pk;auto;column(id)"` + OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + CveNum string `orm:"type(text);column(cve_num);index" description:"cve编号"` + XmlContent string `orm:"type(text);column(xml_content)" description:"xml内容"` + Md5 string `orm:"size(128);column(cur_md5);unique" description:"当前内容生成的MD5"` + UpdateMd5 string `orm:"size(128);column(update_md5)" description:"内容更新后生成的MD5"` + Status int8 `orm:"default(1);column(cve_status)" description:"1:初始化; 2: 已更新; 3: 已删除"` + IsExport int8 `orm:"default(1);column(is_export);null" description:"1: 未导出; 2: 已导出"` + AffectFlag int8 `orm:"default(1);column(affect_flag)" description:"1:受影响标识; 2: 不受影响标识; 3: 其他"` + SortOpenEulerSANum string `orm:"size(128);column(sort_openeuler_sa_num);unique" description:"排序后的安全公告"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SaNumber struct { + SaId int64 `orm:"pk;auto;column(sa_id)"` + OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告完整编号"` + SaYears string `orm:"size(16);column(sa_years);" description:"sa对应的年份"` + SaNum int64 `orm:"default(0);column(sa_num)" description:"sa编号"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已使用"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SaFileList struct { + FileId int64 `orm:"pk;auto;column(file_id)"` + FileName string `orm:"size(512);column(file_name)" description:"sa的cvrf文件名称"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type IssueStatisticsMailList struct { + Id int64 `orm:"pk;auto;column(id)"` + EmailName string `orm:"size(256);column(email_name);unique" description:"收件人"` + EmailType int8 `orm:"default(1);column(email_type)" description:"1:收件人;2:抄送人"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SaFileRecord struct { + FileId int64 `orm:"pk;auto;column(file_id)"` + FileName string `orm:"size(512);column(file_name)" description:"sa的cvrf文件名称"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + SaNumber int64 `orm:"column(sa_number)" description:"当前数字"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenGaussSiteList struct { + GaussId int64 `orm:"pk;auto;column(gauss_id)"` + GaussSaNum string `orm:"size(64);column(gauss_sa_num);unique" description:"opengauss的sa编号"` + SaNum int64 `orm:"null;column(sa_num)" description:"sa对应的数据值"` + Introduction string `orm:"size(256);null;column(introduction)" description:"简介"` + CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + CveLevelValue int `orm:"default(0);column(cve_level_value)" desription:"Critical: 1, High: 2, Medium: 3,low: 4"` + Summary string `orm:"size(256);null;column(summary)" description:"概要"` + Theme string `orm:"type(text);null;column(theme)" description:"主题"` + Description string `orm:"size(8192);column(description)" description:"安全公告描述"` + InfluenceComponent string `orm:"size(256);null;column(influence_component)" description:"影响组件"` + AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` + CveNums string `orm:"type(text);column(cve_nums)" description:"cve编号汇总"` + ReferenceLink string `orm:"type(text);null;column(reference_link)" description:"参考链接"` + PublicDate string `orm:"size(32);null;column(public_date)" description:"sa发布日期"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除;3:已发布"` + Years int `orm:"null;column(sa_years)" description:"sa 发布年份"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenGaussCveList struct { + Id int64 `orm:"pk;auto"` + GaussId int64 `orm:"index;column(gauss_id)"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + RepoName string `orm:"size(512);column(repo_name)" description:"cve对应得仓名称"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` + OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` + NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` + OvectorVule string `orm:"size(256);column(o_vector_value)" description:"openeuler vector 原始值"` + NattackVector string `orm:"size(64);column(n_attack_vector)"` + NaccessVector string `orm:"size(256);column(n_access_vector);null" description:"nvd 2.0 评分向量"` + OattackVector string `orm:"size(64);column(o_attack_vector)"` + OaccessVector string `orm:"size(256);column(o_access_vector);null" description:"nvd 2.0 评分向量"` + NattackComplexity string `orm:"size(64);column(n_attack_complexity)"` + NaccessComplexity string `orm:"size(64);column(n_access_complexity);null"` + OattackComplexity string `orm:"size(64);column(o_attack_complexity)"` + OaccessComplexity string `orm:"size(64);column(o_access_complexity);null"` + NprivilegeRequired string `orm:"size(64);column(n_privilege_required)"` + OprivilegeRequired string `orm:"size(64);column(o_privilege_required)"` + NuserInteraction string `orm:"size(64);column(n_user_interaction)"` + OuserInteraction string `orm:"size(64);column(o_user_interaction)"` + Nscope string `orm:"size(64);column(n_scope)"` + Oscope string `orm:"size(64);column(o_scope)"` + Nconfidentiality string `orm:"size(64);column(n_confidentiality)"` + Oconfidentiality string `orm:"size(64);column(o_confidentiality)"` + Nintegrity string `orm:"size(64);column(n_integrity)"` + Ointegrity string `orm:"size(64);column(o_integrity)"` + Navailability string `orm:"size(64);column(n_availability)"` + Oavailability string `orm:"size(64);column(o_availability)"` + Nauthentication string `orm:"size(64);column(n_authentication);null"` + Oauthentication string `orm:"size(64);column(o_authentication);null"` + ScoreType string `orm:"size(16);column(score_type)"` + FixLabel string `orm:"size(32);column(fix_label)" description:"issue标签, Fixed, Unaffected"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用(预发布);2:已删除;3:正常发布,数据在官网展示"` + Years int `orm:"null;column(cve_years)" description:"cve 发布年份"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenGaussGroup struct { + GroupId int64 `orm:"pk;auto;column(group_id)"` + GroupName string `orm:"size(64);column(group_name);unique" description:"软件包分类组名称"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` +} + +type OpenGaussPackage struct { + PackId int64 `orm:"pk;auto;column(pack_id)"` + GroupId int64 `orm:"index;column(group_id)"` + PlatId int64 `orm:"index;column(plat_id)"` + GaussId int64 `orm:"index;column(gauss_id);default(0)" description:"对应sa的表id"` + PackageName string `orm:"size(512);column(package_name)" description:"cve对应得包名称"` + RpmName string `orm:"type(text);column(rpm_name)" description:"tag包名称"` + ReleaseDate string `orm:"size(32);column(release_date)" description:"版本发布日期"` + RepoVersion string `orm:"size(32);column(repo_version);" description:"分支版本信息"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenGaussDownloadFile struct { + FileId int64 `orm:"pk;auto;column(file_id)"` + FilePath string `orm:"size(512);column(file_path);unique" description:"文件下载路径"` + RepoVersion string `orm:"size(32);column(repo_version);" description:"分支版本信息"` + Status int8 `orm:"default(1);column(status)" description:"1:文件待使用;2:文件解析中;3:文件已废弃"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenGaussVersion struct { + VersionId int64 `orm:"pk;auto;column(version_id)"` + CveId int64 `orm:"index;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + AffectStatus int8 `orm:"default(1);column(affect_status)" description:"1:受影响,Fixed;2:不受影响,Unaffected"` + AffectStatusName string `orm:"size(32);column(affect_status_name);" description:"是否受影响名称, Fixed, Unaffected"` + RepoVersion string `orm:"size(32);column(repo_version);" description:"分支版本信息"` + ReleaseFlag int8 `orm:"default(1);column(release_flag)" description:"1: 受影响标识;2:不受影响标识;3:丢弃"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` +} + +type OpenGaussPlatform struct { + PlatId int64 `orm:"pk;auto;column(plat_id)"` + PlatName string `orm:"size(64);column(plat_name);unique" description:"软件包平台名称"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` +} + +type OpenGaussListTemp struct { + GaussTempId int64 `orm:"pk;auto;column(gauss_temp_id)"` + Introduction string `orm:"size(256);null;column(introduction)" description:"简介"` + CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + CveLevelValue int `orm:"default(0);column(cve_level_value)" desription:"Critical: 1, High: 2, Medium: 3,low: 4"` + Summary string `orm:"size(256);null;column(summary)" description:"概要"` + Theme string `orm:"type(text);null;column(theme)" description:"主题"` + Description string `orm:"type(text);column(description)" description:"安全公告描述"` + InfluenceComponent string `orm:"size(256);null;column(influence_component)" description:"影响组件"` + AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` + PublicDate string `orm:"size(32);null;column(public_date)" description:"sa发布日期"` + CveNums string `orm:"type(text);column(cve_nums)" description:"cve编号汇总"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OpenEulerRepoOrigin struct { + Id int64 `orm:"pk;auto;column(id)"` + PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Version string `orm:"size(64);column(version)" description:"版本号"` + Summary string `orm:"type(text);column(summary)" description:"简介"` + Decription string `orm:"type(text);column(decription)" description:"描述"` + Branchs string `orm:"size(512);column(branch)" description:"分支信息"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` + Repo string `orm:"column(repo_name);size(256)" description:"仓名称"` +} + +type OpenGaussSecurityReviewer struct { + Id int64 `orm:"pk;auto"` + NameSpace string `orm:"unique" description:"码云空间地址"` + Status int8 `orm:"default(0);column(status)" description:"0: 全部；1:审核人"` +} + +type MindSporeSecurityReviewer struct { + Id int64 `orm:"pk;auto"` + NameSpace string `orm:"unique" description:"码云空间地址"` + Status int8 `orm:"default(0);column(status)" description:"0: 全部；1:审核人"` +} + +type MindSporeYaml struct { + Id int64 `orm:"pk;auto"` + Repo string `orm:"column(repo_name);size(256)" description:"仓名称"` + PackageName string `orm:"column(package_name);size(256)" description:"第三方包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` + MarkBit int8 `orm:"default(1);column(mark_bit)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type MindSporeBrandTags struct { + Id int64 `orm:"pk;auto"` + PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Brand string `orm:"size(64);column(brand);index" description:"分支信息"` + Tags string `orm:"size(64);column(tags);index" description:"tags信息"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type OriginUpstreamRecord struct { + CveId int64 `orm:"pk;auto;column(cve_record_id)"` + Ids string `orm:"size(256);column(cve_un_ids)" description:" 唯一编号,根据此字段去重数据, 唯一识别码，可以填cve编号"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + UpdateType string `orm:"size(32);column(update_type);" description:"数据上传类型：insert, update, delete"` + CvePackName string `orm:"size(512);column(cve_packname);null" description:"Cve在上游对应的包名"` + PackName string `orm:"size(512);column(git_packname);index;null" description:"Cve对应的openEuler包名称(或者影响的包名)"` + Title string `orm:"type(text);column(cve_title);null" description:"标题"` + AffectProduct string `orm:"size(512);column(affect_porduct);null" description:"Cve影响的组件, 对应"` + CnnvdID string `orm:"size(256);column(cnnvd_id);null" description:"Cnnvd_id"` + CnvdID string `orm:"size(256);column(cnvd_id);null" description:"Cnvd_id"` + PublishedDate string `orm:"size(32);column(published_date);null" description:"漏洞发布日期"` + FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` + FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` + VulStatus string `orm:"size(64);column(vul_status);null" description:"漏洞状态，REJECT, DISPUTED"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化； + 2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` + AffectedScope string `orm:"size(512);column(affected_scope);null" description:"影响范围推理"` + Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` + AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` + IsExit int8 `orm:"default(1);column(is_exit)" description:"1: 当前包对应在src-openEuler有对应仓库; 0: 无;2:临时值"` + Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; + 1:包名通过别名匹配、版本号为漏洞库原始数据; + 2:包名版本号都通过漏洞描述获取; + 3:包名通过漏洞描述获取并通过别名匹配、版本号为漏洞描述获取; + 4:包名版本号通过SA获取、版本号为修复版本号、修复版本以下都视为受影响版本; + 5:包名版本号通过SA获取并通过别名匹配、版本号为修复版本号、修复版本以下都视为受影响版本; + 6:版本号未匹配、包名通过以上任意方式匹配、可信度最低"` + OrgData string `orm:"type(text);column(org_data);null" description:"原始数据"` + CreateTime string `orm:"size(32);column(create_time)"` + SysStatus int8 `orm:"default(0);column(sys_status)" description:"0:cve新增;1:数据已变化； + 2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` + ErrDesc string `orm:"null;type(text);column(err_desc);null" description:"数据同步失败的错误描述"` +} + +type OpenLookengSecurityReviewer struct { + Id int64 `orm:"pk;auto"` + NameSpace string `orm:"unique" description:"码云空间地址"` + Status int8 `orm:"default(0);column(status)" description:"0: 全部；1:审核人"` +} + +type OpenLookengYaml struct { + Id int64 `orm:"pk;auto"` + Repo string `orm:"column(repo_name);size(256)" description:"仓名称"` + PackageName string `orm:"column(package_name);size(256)" description:"第三方包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` + MarkBit int8 `orm:"default(1);column(mark_bit)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type IssueCommunityStatistics struct { + Id int64 `orm:"pk;auto"` + CveId int64 `orm:"column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num);unique" description:"cve编号"` + Repo string `orm:"column(repo_name);size(256)" description:"仓名称"` + PackageName string `orm:"column(package_name);size(256)" description:"组件名称"` + Version string `orm:"size(64);column(version)" description:"版本号"` + ExistFlag int8 `orm:"default(1);column(exist_flag)" description:"1:openEuler;2:openGauss;3:MindSpore;4:openLooKeng"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type CommunityYamlConfig struct { + Id int64 `orm:"pk;auto"` + TableName string `orm:"column(table_name);size(512)" description:"表名称"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` +} + +type IssueDeleteRecord struct { + IssueDeleteId int64 `orm:"pk;auto;column(id)"` + IssueId int64 `orm:"column(issue_id)" description:"issue的id"` + IssueNum string `orm:"size(64);column(issue_num)" description:"issue编号"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` + DelAssignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"` + IssueLabel string `orm:"size(256);column(issue_label)" description:"issue标签, CVE/FIXED, CVE/UNFIXED"` + Owner string `orm:"size(128);column(owner)" description:"仓库地址"` + Repo string `orm:"size(512);column(repo)" description:"仓库路径"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +func CreateDb() bool { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return false + } + prefix := BConfig.String("mysql::dbprefix") + InitdbType, _ := beego.AppConfig.Int("initdb") + if InitdbType == 1 { + orm.RegisterModelWithPrefix(prefix, new(OtherUser), new(AdminUser), new(VulnCenter), + new(OpenEulerSA), new(Score), new(SecurityNotice), new(Package), + new(IssueTemplate), new(ScoreRecord), new(IssueHooks), + new(GitPackageTable), new(GitOpenEuler), new(GitPackageInfo), + new(GitSubPack), new(GitOpenEulerTableRelate), + new(SecurityReviewer), new(GitSubPackRequire), + new(GitSubPackProvides), new(GitSubPackRequiredby), + new(IpWhite), new(OriginUpstream), new(OriginUpstreamDesc), + new(OriginUpstreamConfig), new(OriginUpstreamConfigNode), + new(OriginUpstreamConfigNodeCpe), new(OriginUpstreamImpact), new(OriginUpstreamImpactScore), + new(OriginUpstreamImpactScoreV3), new(OriginUpstreamImpactScoreV2), + new(OriginUpstreamPoc), new(OriginUpstreamEvent), + new(OriginUpstreamReference), new(OriginUpstreamVulType), + new(OriginUpstreamFixSuggest), new(OriginUpstreamFixSuggestRefTag), + new(OriginUpstreamFixSuggestRef), new(GiteOriginIssue), + new(OriginExcel), new(ExportRecord), new(GitRepoGroups), + new(GiteRepo), new(GiteRepoMember), new(GiteRepoBranch), + new(PackageCpe), new(EmailList), new(IssueAssignee), + new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), + new(SpecIssueAssigness), new(SpecError), new(CvrfSaRecord), + new(SaNumber), new(SaFileList), new(IssueStatisticsMailList), + new(SaFileRecord), new(OpenGaussSiteList), new(OpenGaussCveList), + new(OpenGaussGroup), new(OpenGaussPackage), + new(IssueTemplateAssociation), + new(OpenGaussDownloadFile), new(OpenGaussVersion), + new(OpenGaussPlatform), new(OpenGaussListTemp), + new(OpenEulerRepoOrigin), new(OpenGaussSecurityReviewer), + new(MindSporeYaml), new(MindSporeSecurityReviewer), + new(MindSporeBrandTags), new(OriginUpstreamRecord), + new(OpenLookengSecurityReviewer), new(OpenLookengYaml), + new(IssueCommunityStatistics), new(CommunityYamlConfig), + new(IssueDeleteRecord), + ) + logs.Info("table create success!") + errosyn := orm.RunSyncdb("default", false, true) + if errosyn != nil { + logs.Error(errosyn) + } + } + return true +} diff --git a/cve-vulner-manager/models/object.go b/cve-vulner-manager/models/object.go new file mode 100644 index 0000000000000000000000000000000000000000..3ffb60850f104eabf7ca3d29fda97cf49c258d16 --- /dev/null +++ b/cve-vulner-manager/models/object.go @@ -0,0 +1,53 @@ +package models + +import ( + "errors" + "strconv" + "time" +) + +var ( + Objects map[string]*Object +) + +type Object struct { + ObjectId string + Score int64 + PlayerName string +} + +func init() { + Objects = make(map[string]*Object) + Objects["hjkhsbnmn123"] = &Object{"hjkhsbnmn123", 100, "astaxie"} + Objects["mjjkxsxsaa23"] = &Object{"mjjkxsxsaa23", 101, "someone"} +} + +func AddOne(object Object) (ObjectId string) { + object.ObjectId = "astaxie" + strconv.FormatInt(time.Now().UnixNano(), 10) + Objects[object.ObjectId] = &object + return object.ObjectId +} + +func GetOne(ObjectId string) (object *Object, err error) { + if v, ok := Objects[ObjectId]; ok { + return v, nil + } + return nil, errors.New("ObjectId Not Exist") +} + +func GetAll() map[string]*Object { + return Objects +} + +func Update(ObjectId string, Score int64) (err error) { + if v, ok := Objects[ObjectId]; ok { + v.Score = Score + return nil + } + return errors.New("ObjectId Not Exist") +} + +func Delete(ObjectId string) { + delete(Objects, ObjectId) +} + diff --git a/cve-vulner-manager/models/oricvecheck.go b/cve-vulner-manager/models/oricvecheck.go new file mode 100644 index 0000000000000000000000000000000000000000..24eb4c63c61dd4149caf033fbf0c740c040989ce --- /dev/null +++ b/cve-vulner-manager/models/oricvecheck.go @@ -0,0 +1,76 @@ +package models + +import ( + "cvevulner/common" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +func QueryOriginCveE(prcnum int) ([]OriginUpstream, int64, error) { + o := orm.NewOrm() + var os []OriginUpstream + num, err := o.Raw("select cve_id,cve_un_ids,cve_num, update_type,cve_packname,"+ + "git_packname,cve_title,affect_porduct,cnnvd_id,cnvd_id,published_date,vul_status,cve_status,version"+ + " from cve_origin_upstream where cve_status in (?, ?) and is_exit = ? "+ + "order by cve_id asc limit ?", 0, 1, 2, prcnum).QueryRows(&os) + if err == nil && num > 0 { + logs.Info("cve_origin_upstream, Number of items found: ", num) + } else { + logs.Info("QueryOriginCveE, There is currently no new or updated cve, cur_time:", + common.GetCurTime(), ", err: ", err) + } + return os, num, err +} + +func UpdateOriginExist(updatetime, pakName, version string, cveId int64, isExit int) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "is_exit = ?, update_time = ? where cve_id = ? and git_packname = ? and version = ?", + isExit, updatetime, cveId, pakName, version).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("cve_origin_upstream row affected nums: ", num, + ",cveId: ", cveId, ",", updatetime, pakName, version) + return true + } + return false + } else { + logs.Error("UpdateOriginExist, Status update failed, cveId: ", + cveId, ",", updatetime, pakName, version, ", err: ", err) + return false + } +} + +func UpdateOriginExistTemp() (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "is_exit = ? where is_exit = ?", 2, 0).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("cve_origin_upstream row affected nums: ", num) + return true + } + return false + } else { + logs.Error("UpdateOriginExistTemp, Update failed, err: ", err) + return false + } +} + +// Query abnormal cve data +func QueryAbnCve(cveSt, days string, prcnum int, cveId int64) ([]VulnCenter, error) { + o := orm.NewOrm() + var vc []VulnCenter + cveSql := fmt.Sprintf("select * from cve_vuln_center where "+ + "update_time >= '%s' and cve_id > %d and cve_status in (%s) "+ + "order by cve_id asc limit %d", days, cveId, cveSt, prcnum) + logs.Info("cveSql: ", cveSql) + num, err := o.Raw(cveSql).QueryRows(&vc) + if err == nil && num > 0 { + return vc, err + } + return vc, err +} diff --git a/cve-vulner-manager/models/packages.go b/cve-vulner-manager/models/packages.go new file mode 100644 index 0000000000000000000000000000000000000000..78b556d77bf6c5d1cbcd91f02c4e938518d0a1e0 --- /dev/null +++ b/cve-vulner-manager/models/packages.go @@ -0,0 +1,148 @@ +package models + +import ( + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +func GetIpWhite(ip string, iw *IpWhite) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_ip_white WHERE machine_ip=?", ip).QueryRow(iw) + if err == nil { + logs.Info("GetIpWhite, ip:", ip, " ,In the whitelist") + } else { + // Do not process blacklist + //logs.Error("ip:", ip, " Restricted access") + } + return nil +} + +func GetPackageInfo(packageName, version string, gi *GitPackageInfo) (err error) { + o := orm.NewOrm() + err = o.Raw("select * FROM cve_git_package_info WHERE package_name=? and version=? and status=?", + packageName, version, 0).QueryRow(gi) + if err == nil { + logs.Info("GetPackageInfo, cve_git_package_info, "+ + "packageName: ", packageName, ", search successful") + return nil + } else { + logs.Error("GetPackageInfo, cve_git_package_info, "+ + "packageName: ", packageName, ", Data does not exist, err: ", err) + } + return err +} + +func GetSubPackage(packageInfoId int64) (gg []GitSubPack, num int64, err error) { + o := orm.NewOrm() + var gs []GitSubPack + num, err = o.Raw("select sub_id,ids,sub_pack_name "+ + "FROM cve_git_sub_pack WHERE detail_id=?", + packageInfoId).QueryRows(&gs) + if num > 0 && err == nil { + logs.Info("GetSubPackage, cve_git_sub_pack, "+ + "packageInfoId: ", packageInfoId, " , search successful") + return gs, num, nil + } else { + logs.Error("GetSubPackage, cve_git_sub_pack, "+ + "packageInfoId: ", packageInfoId, " , Data does not exist, err: ", err) + } + return gs, 0, err +} + +func GetSubPackProvide(subId int64) (gg []GitSubPackProvides, num int64, err error) { + o := orm.NewOrm() + var gs []GitSubPackProvides + num, err = o.Raw("select provide_id,ids,provide_name, requiredb "+ + "FROM cve_git_sub_pack_provides WHERE sub_id=?", + subId).QueryRows(&gs) + if num > 0 && err == nil { + logs.Info("GetSubPackProvide, cve_git_sub_pack_provides, "+ + "subId: ", subId, " , search successful") + return gs, num, nil + } else { + logs.Error("GetSubPackProvide, cve_git_sub_pack_provides, "+ + "subId: ", subId, " , Data does not exist, err: ", err) + } + return gs, 0, err +} + +func GetSubPackrequiredby(provideId int64) (gg []GitSubPackRequiredby, num int64, err error) { + o := orm.NewOrm() + var gs []GitSubPackRequiredby + num, err = o.Raw("select id,requiredb FROM cve_git_sub_pack_requiredby WHERE provide_id=?", + provideId).QueryRows(&gs) + if num > 0 && err == nil { + logs.Info("GetSubPackrequiredby, cve_git_sub_pack_requiredby, "+ + "provideId: ", provideId, " , search successful") + return gs, num, nil + } else { + logs.Error("GetSubPackrequiredby, cve_git_sub_pack_requiredby, "+ + "provideId: ", provideId, " , Data does not exist, err: ", err) + } + return gs, 0, err +} + +func GetSubPackRequire(subId int64) (gg []GitSubPackRequire, num int64, err error) { + o := orm.NewOrm() + var gs []GitSubPackRequire + num, err = o.Raw("select require_id,ids,require_name, providedb "+ + "FROM cve_git_sub_pack_require WHERE sub_id=?", + subId).QueryRows(&gs) + if num > 0 && err == nil { + logs.Info("GetSubPackRequire, cve_git_sub_pack_require, "+ + "subId: ", subId, " , search successful") + return gs, num, nil + } else { + logs.Error("GetSubPackRequire, cve_git_sub_pack_require, "+ + "subId: ", subId, " , Data does not exist, err: ", err) + } + return gs, 0, err +} + +func GetPackageList(pagesize int64, pagenum int64, QueryPkgName string) (ge []GitOpenEuler, num int64, err error) { + o := orm.NewOrm() + qs := o.QueryTable("cve_git_open_euler") + var us []GitOpenEuler + if QueryPkgName != "" { + cnt, err := qs.Filter("package_name__icontains", QueryPkgName).Filter("status", 1).Limit(pagesize, (pagenum-1)*pagesize).All(&us) + if err == nil { + logs.Info("GetPackageList, cve_git_open_euler, count: ", cnt, ",pagesize ", + pagesize, ",pagenum ", pagenum, ", QueryPkgName ", QueryPkgName) + } else { + logs.Error("GetPackageList, cve_git_open_euler, err: ", err, ",pagesize ", + pagesize, ",pagenum ", pagenum, ", QueryPkgName ", QueryPkgName) + } + return us, cnt, err + } else { + cnt, err := qs.Filter("status", 1).Limit(pagesize, (pagenum-1)*pagesize).All(&us) + if err == nil { + logs.Info("GetPackageList, cve_git_open_euler, count: ", cnt, ",pagesize ", + pagesize, ",pagenum ", pagenum, ", QueryPkgName ", QueryPkgName) + } else { + logs.Error("GetPackageList, cve_git_open_euler, err: ", err, ",pagesize ", + pagesize, ",pagenum ", pagenum, ", QueryPkgName ", QueryPkgName) + } + return us, cnt, err + } +} + +func GetPackageNum(QueryPkgName string) int64 { + o := orm.NewOrm() + cg := o.QueryTable("cve_git_open_euler") + var ge []GitOpenEuler + if QueryPkgName != "" { + num, err := cg.Filter("package_name__icontains", QueryPkgName).Filter("status", 1).All(&ge) + if err == nil { + return num + } else { + return 0 + } + } else { + num, err := cg.Filter("git_id__gt", 0).Filter("status", 1).All(&ge) + if err == nil { + return num + } else { + return 0 + } + } +} diff --git a/cve-vulner-manager/models/reviewer.go b/cve-vulner-manager/models/reviewer.go new file mode 100644 index 0000000000000000000000000000000000000000..1739545b7e55c1f6fa139dcaac0f6509aab25b5f --- /dev/null +++ b/cve-vulner-manager/models/reviewer.go @@ -0,0 +1,70 @@ +package models + +import ( + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +func (s *SecurityReviewer) Read(filed ...string) (ok bool) { + o := orm.NewOrm() + err := o.Read(s, filed...) + if err != nil { + logs.Error(err) + return false + } + return true +} + +func (s *OpenGaussSecurityReviewer) Read(filed ...string) (ok bool) { + o := orm.NewOrm() + err := o.Read(s, filed...) + if err != nil { + logs.Error(err) + return false + } + return true +} + +func GetSecurityReviewerList() (list []SecurityReviewer, err error) { + o := orm.NewOrm() + _, err = o.QueryTable("cve_security_reviewer").All(&list) + return list, err +} + +func GetGuassSecurityReviewerList() (list []OpenGaussSecurityReviewer, err error) { + o := orm.NewOrm() + _, err = o.QueryTable("cve_open_gauss_security_reviewer").All(&list) + return list, err +} + +func GetSporeSecurityReviewerList() (list []MindSporeSecurityReviewer, err error) { + o := orm.NewOrm() + _, err = o.QueryTable("cve_mind_spore_security_reviewer").All(&list) + return list, err +} + +func GetOpenLookengSecurityReviewerList() (list []OpenLookengSecurityReviewer, err error) { + o := orm.NewOrm() + _, err = o.QueryTable("cve_open_lookeng_security_reviewer").All(&list) + return list, err +} + +func (s *MindSporeSecurityReviewer) Read(filed ...string) (ok bool) { + o := orm.NewOrm() + err := o.Read(s, filed...) + if err != nil { + logs.Error(err) + return false + } + return true +} + +func (s *OpenLookengSecurityReviewer) Read(filed ...string) (ok bool) { + o := orm.NewOrm() + err := o.Read(s, filed...) + if err != nil { + logs.Error(err) + return false + } + return true +} \ No newline at end of file diff --git a/cve-vulner-manager/models/securitynotice.go b/cve-vulner-manager/models/securitynotice.go new file mode 100644 index 0000000000000000000000000000000000000000..b1bdaf19e639dd69c7d4a8f913aa7fb25c71d436 --- /dev/null +++ b/cve-vulner-manager/models/securitynotice.go @@ -0,0 +1,15 @@ +package models + +import "github.com/astaxie/beego/orm" + +//Read get security notice by colName +func (sn *SecurityNotice) Read(colName ...string) error { + o := orm.NewOrm() + return o.Read(sn, colName...) +} +//Update update security notice by column names +func (sn *SecurityNotice)Update(colName ...string) error { + o := orm.NewOrm() + _, err := o.Update(sn, colName...) + return err +} diff --git a/cve-vulner-manager/models/spore.go b/cve-vulner-manager/models/spore.go new file mode 100644 index 0000000000000000000000000000000000000000..919ae3f608c71f8b025ac286ff4ab41af2f2e6d3 --- /dev/null +++ b/cve-vulner-manager/models/spore.go @@ -0,0 +1,35 @@ +package models + +import ( + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +// delete data +func DelMindSporeBrandTagsByBrand(brand, packageName string) { + o := orm.NewOrm() + err := o.Raw("delete from cve_mind_spore_brand_tags where brand = ? and package_name = ?", + brand, packageName).QueryRow() + logs.Info("DelMindSporeBrandTagsByBrand", err) +} + +// insert data +func InsertMindSporeBrandTags(mbt *MindSporeBrandTags) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(mbt) + return id, err +} + +func QueryMindSporeBrandTags(osl *MindSporeBrandTags, field ...string) error { + o := orm.NewOrm() + err := o.Read(osl, field...) + return err +} + +func QueryMindSporeTagsByBrand(osl *MindSporeBrandTags) (mbt []MindSporeBrandTags) { + o := orm.NewOrm() + _, err := o.Raw("SELECT * from cve_mind_spore_brand_tags WHERE package_name = ? and brand = ?", + osl.PackageName, osl.Brand).QueryRows(&mbt) + logs.Info("QueryMindSporeTagsByBrand, err: ", err) + return +} diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go new file mode 100644 index 0000000000000000000000000000000000000000..c870a53c75e09946ef6fc37aad85861ce0578baf --- /dev/null +++ b/cve-vulner-manager/models/uploadcve.go @@ -0,0 +1,1174 @@ +package models + +import ( + "cvevulner/common" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "strings" +) + +type OpenSaId struct { + Id int64 +} + +func QueryYamlData(ge *GitOpenEuler, colName ...string) error { + o := orm.NewOrm() + err := o.Read(ge, colName...) + return err +} + +// insert data +func InsertYamlData(ge *GitOpenEuler) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ge) + return id, err +} + +func UpdateYamlData(ge *GitOpenEuler, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ge, fields...) + return err +} + +func QueryGitPackageInfo(ge *GitPackageInfo, colName ...string) error { + o := orm.NewOrm() + err := o.Read(ge, colName...) + return err +} + +// insert data +func InsertGitPackageInfo(ge *GitPackageInfo) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ge) + return id, err +} + +func UpdateGitPackageInfo(ge *GitPackageInfo, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ge, fields...) + return err +} + +func QueryOriginRepo(pkgName string) (GiteRepo, bool) { + o := orm.NewOrm() + var gr GiteRepo + repoOrg, errx := common.GetRepoOrg() + if errx != nil { + logs.Error("QueryOriginRepo, GetRepoOrg, pkgName: ", pkgName, + ",Unable to get the organization corresponding to the repo, err: ", errx) + return gr, false + } + err := o.Raw("select * from cve_gite_repo where path = ? and org_path = ? and status = ?", + pkgName, repoOrg, 0).QueryRow(&gr) + if err != nil { + logs.Error("QueryOriginRepo, cve_gite_repo, "+ + "pkgName:", pkgName, ", repoOrg: ", repoOrg, ", No corresponding data") + return gr, false + } else { + logs.Info("QueryOriginRepo, cve_gite_repo, "+ + "pkgName: ", pkgName, ", repoOrg: ", repoOrg, ", search successful") + return gr, true + } +} + +func QueryOrgAllRepo(org string) ([]GiteRepo) { + o := orm.NewOrm() + var gr []GiteRepo + num, err := o.Raw("select * from cve_gite_repo where org_path = ? and status = ?", org, 0).QueryRows(&gr) + if num == 0 { + logs.Info("QueryOrgAllRepo, cve_gite_repo"+", repoOrg: ", org, ", No corresponding data, err: ", err) + return gr + } else { + logs.Info("QueryOrgAllRepo, cve_gite_repo"+", repoOrg: ", org, ", search successful") + return gr + } +} + +func QueryRepoMember(groupId int64, memberType string) ([]GiteRepoMember, bool) { + o := orm.NewOrm() + var grm []GiteRepoMember + num, err := o.Raw("select * from cve_gite_repo_member where group_id = ?", + groupId).QueryRows(&grm) + if err != nil || num == 0 { + logs.Info("QueryRepoMember, cve_gite_repo_member, "+ + "groupId: ", groupId, ",memberType: ", memberType, ", No corresponding data") + return grm, false + } else { + logs.Info("QueryRepoMember, cve_gite_repo_member, "+ + "groupId: ", groupId, ",memberType: ", memberType, ", search successful") + return grm, true + } +} + +func QueryRepoAllMaintainer(pkgName string) ([]GiteRepoMember, bool) { + sia := SpecIssueAssigness{PackageName: pkgName, Status: 1} + gr, okrg := QueryOriginRepo(pkgName) + if !okrg || gr.RepoId == 0 { + logs.Error("QueryRepoAllMaintainer, maintainer query failed, "+ + "pkgName: ", pkgName, ",okrg: ", okrg) + specError := GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + var grm []GiteRepoMember + var grmr GiteRepoMember + grmr.MemberName = sia.Assignee + grm = append(grm, grmr) + return grm, true + } + return nil, false + } + grm, okgrm := QueryRepoMember(gr.GroupId, "Maintainer") + if !okgrm { + logs.Error("QueryRepoMember, maintainer query failed, "+ + "pkgName: ", pkgName, ",gr.GroupId: ", gr.GroupId, ",okgrm: ", okgrm) + specError := GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + var grmr GiteRepoMember + grmr.MemberName = sia.Assignee + grm = append(grm, grmr) + return grm, true + } + return nil, false + } + specError := GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + assFlag := false + for _, mt := range grm { + if mt.MemberName == sia.Assignee { + assFlag = true + break + } + } + if !assFlag { + var grmr GiteRepoMember + grmr.MemberName = sia.Assignee + grm = append(grm, grmr) + } + } + return grm, true +} + +func QueryRepoMaintainer(pkgName string) (string, bool) { + gr, okrg := QueryOriginRepo(pkgName) + if !okrg || gr.RepoId == 0 { + logs.Error("QueryRepoMaintainer, repo does not exist, err: ", okrg) + return "", okrg + } + grm, okgrm := QueryRepoMember(gr.GroupId, "Maintainer") + if !okgrm { + logs.Error("QueryRepoMember, Maintainer does not exist, err: ", okrg) + return "", okgrm + } + for _, g := range grm { + if g.MemberName != "" && len(g.MemberName) > 1 { + return g.MemberName, true + } + } + return "", false +} + +func QueryCveOpeneulerdata(ge *GitOpenEuler) bool { + geErr := QueryYamlData(ge, "PackageName", "Version", "Status") + if ge.GitId == 0 { + logs.Error("QueryCveOpeneulerdata, There is no corresponding repo or version, ", + ge.PackageName, ge.Version, ",geErr: ", geErr) + return false + } + return true +} + +func QueryCveOpeneulerDetaildataByName(pkgName, versions string) (goe GitPackageInfo, bl bool) { + o := orm.NewOrm() + err := error(nil) + if versions == "" || len(versions) == 0 { + err = o.Raw("select * from cve_git_package_info where package_name = ? and status = ? "+ + "order by detail_id desc limit 1", pkgName, 0).QueryRow(&goe) + } else { + err = o.Raw("select * from cve_git_package_info where package_name = ? and version = ? and status = ? "+ + "order by detail_id desc limit 1", pkgName, versions, 0).QueryRow(&goe) + } + if err != nil { + logs.Error("QueryCveOpeneulerDetaildataByName, cve_git_package_info, pkgName: ", pkgName, ", versions: ", versions, + ", No corresponding data, err: ", err) + return goe, false + } else { + logs.Info("QueryCveOpeneulerDetaildataByName, cve_git_package_info, pkgName: ", pkgName, ", versions: ", versions, + ", search successful") + return goe, true + } +} + +func QueryCveOriginByIds(ids string) (OriginUpstream, bool) { + o := orm.NewOrm() + var orcve OriginUpstream + err := o.Raw("select * from cve_origin_upstream where cve_un_ids = ?", ids).QueryRow(&orcve) + if err != nil { + return orcve, false + } else { + return orcve, true + } +} + +func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *OriginUpstreamDesc, + ous *OriginUpstreamConfig, osi *OriginUpstreamImpact, osp *OriginUpstreamPoc, ose *OriginUpstreamEvent, + osv *OriginUpstreamVulType, osf *OriginUpstreamFixSuggest) (Id int64, err error) { + o := orm.NewOrm() + errs := o.Begin() + if errs == nil { + ouse := OriginUpstream{Ids: ou.Ids} + err := o.Read(&ouse, "Ids") + if err == orm.ErrNoRows || err == orm.ErrMissPK { + logs.Info("CreateOriginCve, Add data: ", ou) + var num int64 + if num, err = o.Insert(ou); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream success, num:, cveNum", num, ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream failed, ou:", ou, ", err: ", err) + o.Rollback() + return 0, err + } + od.CveId = num + lod := OriginUpstreamDesc{CveId: num} + o.Delete(&lod, "CveId") + if odnum, err := o.Insert(od); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_desc success, num:", odnum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_desc failed, ou:", ou, ", err: ", err) + o.Rollback() + return 0, err + } + ous.CveId = num + lous := OriginUpstreamConfig{CveId: num} + err := o.Read(&lous, "CveId") + if err == orm.ErrNoRows || err == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_config, "+ + "Data does not exist, cveId: ", num) + } else { + var lousc []OriginUpstreamConfigNode + louscNum, err := o.Raw("select * from cve_origin_upstream_config_node where conf_id = ?", + lous.ConfId).QueryRows(&lousc) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_config_node, "+ + "Data does not exist, louscNum: ", louscNum) + } else { + for _, lsc := range lousc { + ousnc := OriginUpstreamConfigNodeCpe{NodeId: lsc.NodeId} + o.Delete(&ousnc, "NodeId") + } + ousn := OriginUpstreamConfigNode{ConfId: lous.ConfId} + o.Delete(&ousn, "ConfId") + } + lousn := OriginUpstreamConfig{CveId: num} + o.Delete(&lousn, "CveId") + } + if ousnum, err := o.Insert(ous); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_config success, "+ + "ousnum:", ousnum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_config failed,"+ + " ou:", ous, ", err: ", err) + o.Rollback() + return 0, err + } + if CveData.Configurations.Nodes != nil && len(CveData.Configurations.Nodes) > 0 { + for _, nodes := range CveData.Configurations.Nodes { + var Lnode OriginUpstreamConfigNode + Lnode.Operator = nodes.Operator + Lnode.ConfId = ous.ConfId + if lousnum, err := o.Insert(&Lnode); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_config_node success, "+ + "lousnum:", lousnum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_config_node failed, "+ + "Lnode:", Lnode, ", err: ", err) + o.Rollback() + return 0, err + } + if nodes.Cpe != nil && len(nodes.Cpe) > 0 { + for _, nodCpe := range nodes.Cpe { + var ouscnc OriginUpstreamConfigNodeCpe + ouscnc.Cpe23Uri = nodCpe.Cpe23Uri + ouscnc.NodeId = Lnode.NodeId + ouscnc.CpeMatchString = nodCpe.CpeMatchString + ouscnc.Vulnerable = nodCpe.Vulnerable + if lousnumc, err := o.Insert(&ouscnc); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_config_node_cpe success, "+ + "lousnumc:", lousnumc, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_config_node_cpe failed, "+ + "ouscnc:", ouscnc, ", err: ", err) + o.Rollback() + return 0, err + } + } + } + } + } + osi.CveId = num + losi := OriginUpstreamImpact{CveId: num} + errxx := o.Read(&losi, "CveId") + if errxx == orm.ErrNoRows || errxx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_impact, "+ + "Data does not exist, cveId: ", num) + } else { + var losis []OriginUpstreamImpactScore + losisNum, err := o.Raw("select * from cve_origin_upstream_impact_score where impact_id = ?", + losi.ImpactId).QueryRows(&losis) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_impact_score, Data does not exist,"+ + " losi.ImpactId: ", losi.ImpactId, ", err: ", err, ",losisNum: ", losisNum) + } else { + for _, sis := range losis { + if sis.CvssV3 == 1 && sis.BaseMetricV3 == 1 { + lousisv3 := OriginUpstreamImpactScoreV3{ScoreId: sis.ScoreId} + o.Delete(&lousisv3, "ScoreId") + } + if sis.CvssV2 == 1 && sis.BaseMetricV2 == 1 { + lousisv2 := OriginUpstreamImpactScoreV2{ScoreId: sis.ScoreId} + o.Delete(&lousisv2, "ScoreId") + } + } + losisx := OriginUpstreamImpactScore{ImpactId: losi.ImpactId} + o.Delete(&losisx, "ImpactId") + } + losix := OriginUpstreamImpact{CveId: num} + o.Delete(&losix, "CveId") + } + if losinum, err := o.Insert(osi); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact success, lousnum:", + losinum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact failed,"+ + " Lnode:", osi, ", err: ", err) + o.Rollback() + return 0, err + } + var lousist OriginUpstreamImpactScore + lousist.ImpactId = osi.ImpactId + lousist.BaseMetricV3 = 1 + lousist.BaseMetricV2 = 0 + lousist.CvssV3 = 1 + lousist.CvssV2 = 0 + lousist.Status = 1 + if lousistnum, err := o.Insert(&lousist); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ + "lousistnum:", lousistnum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ + "lousist:", lousist, ", err: ", err) + o.Rollback() + return 0, err + } + var lousisv3 OriginUpstreamImpactScoreV3 + lousisv3.ScoreId = lousist.ScoreId + lousisv3.BaseScore = CveData.Impact.BaseMetricV3.CvssV3.BaseScore + lousisv3.VectorString = CveData.Impact.BaseMetricV3.CvssV3.VectorString + lousisv3.AttackComplexity = CveData.Impact.BaseMetricV3.CvssV3.AttackComplexity + lousisv3.AttackVector = CveData.Impact.BaseMetricV3.CvssV3.AttackVector + lousisv3.AvailabilityImpact = CveData.Impact.BaseMetricV3.CvssV3.AvailabilityImpact + lousisv3.BaseSeverity = CveData.Impact.BaseMetricV3.CvssV3.BaseSeverity + lousisv3.UserInteraction = CveData.Impact.BaseMetricV3.CvssV3.UserInteraction + lousisv3.PrivilegesRequired = CveData.Impact.BaseMetricV3.CvssV3.PrivilegesRequired + lousisv3.Version = CveData.Impact.BaseMetricV3.CvssV3.Version + lousisv3.ConfidentialityImpact = CveData.Impact.BaseMetricV3.CvssV3.ConfidentialityImpact + lousisv3.IntegrityImpact = CveData.Impact.BaseMetricV3.CvssV3.IntegrityImpact + lousisv3.Scope = CveData.Impact.BaseMetricV3.CvssV3.Scope + lousisv3.ImpactScore = CveData.Impact.BaseMetricV3.ImpactScore + lousisv3.ExploitabilityScore = CveData.Impact.BaseMetricV3.ExploitabilityScore + lousisv3.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV3.CvssV3.BaseScore) + if lousistnumv3, err := o.Insert(&lousisv3); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 success, "+ + "lousistnumv3:", lousistnumv3, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 failed, "+ + "lousisv3:", lousisv3, ", err: ", err) + o.Rollback() + return 0, err + } + var lousistv2 OriginUpstreamImpactScore + lousistv2.ImpactId = osi.ImpactId + lousistv2.BaseMetricV3 = 0 + lousistv2.BaseMetricV2 = 1 + lousistv2.CvssV3 = 0 + lousistv2.CvssV2 = 1 + lousistv2.Status = 1 + if lousistnum, err := o.Insert(&lousistv2); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ + "lousistnum:", lousistnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ + "lousistv2:", lousistv2, ", err: ", err) + o.Rollback() + return 0, err + } + var lousisv2 OriginUpstreamImpactScoreV2 + lousisv2.ScoreId = lousistv2.ScoreId + lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore + lousisv2.AcInsufInfo = CveData.Impact.BaseMetricV2.AcInsufInfo + lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore + lousisv2.VectorString = CveData.Impact.BaseMetricV2.CvssV2.VectorString + lousisv2.AccessComplexity = CveData.Impact.BaseMetricV2.CvssV2.AccessComplexity + lousisv2.Authentication = CveData.Impact.BaseMetricV2.CvssV2.Authentication + lousisv2.AvailabilityImpact = CveData.Impact.BaseMetricV2.CvssV2.AvailabilityImpact + lousisv2.Version = CveData.Impact.BaseMetricV2.CvssV2.Version + lousisv2.ConfidentialityImpact = CveData.Impact.BaseMetricV2.CvssV2.ConfidentialityImpact + lousisv2.IntegrityImpact = CveData.Impact.BaseMetricV2.CvssV2.IntegrityImpact + lousisv2.AccessVector = CveData.Impact.BaseMetricV2.CvssV2.AccessVector + lousisv2.UserInteractionRequired = CveData.Impact.BaseMetricV2.UserInteractionRequired + lousisv2.Severity = CveData.Impact.BaseMetricV2.Severity + lousisv2.ObtainUserPrivilege = CveData.Impact.BaseMetricV2.ObtainUserPrivilege + lousisv2.ObtainAllPrivilege = CveData.Impact.BaseMetricV2.ObtainAllPrivilege + lousisv2.ObtainOtherPrivilege = CveData.Impact.BaseMetricV2.ObtainOtherPrivilege + lousisv2.ImpactScore = CveData.Impact.BaseMetricV2.ImpactScore + lousisv2.ExploitabilityScore = CveData.Impact.BaseMetricV2.ExploitabilityScore + lousisv2.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV2.CvssV2.BaseScore) + if lousistnumv2, err := o.Insert(&lousisv2); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 success, "+ + "lousistnumv2:", lousistnumv2, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 failed,"+ + " lousisv2:", lousisv2, ", err: ", err) + o.Rollback() + return 0, err + } + + osp.CveId = num + losp := OriginUpstreamPoc{CveId: num} + errxxx := o.Read(&losp, "CveId") + if errxxx == orm.ErrNoRows || errxxx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_impact, Data does not exist, cveId: ", num) + } else { + lospd := OriginUpstreamPoc{CveId: num} + o.Delete(&lospd, "CveId") + } + if ospnum, err := o.Insert(osp); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_poc success, "+ + "ospnum:", ospnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_poc failed, "+ + "osp:", osp, ", err: ", err) + o.Rollback() + return 0, err + } + + ose.CveId = num + lose := OriginUpstreamEvent{CveId: num} + errx := o.Read(&lose, "CveId") + if errx == orm.ErrNoRows || errx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_event, Data does not exist, cveId: ", num) + } else { + losed := OriginUpstreamEvent{CveId: num} + o.Delete(&losed, "CveId") + } + if osenum, err := o.Insert(ose); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_event success, "+ + "osenum:", osenum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_event failed, "+ + "ose:", ose, ", err: ", err) + o.Rollback() + return 0, err + } + + lousr := OriginUpstreamReference{CveId: num} + o.Delete(&lousr, "CveId") + if CveData.ReferenceData != nil && len(CveData.ReferenceData) > 0 { + for _, ref := range CveData.ReferenceData { + var lousrd OriginUpstreamReference + lousrd.CveId = num + lousrd.Url = ref.Url + lousrd.Refsource = ref.Refsource + lousrd.Name = ref.Name + tags := "" + if ref.Tags != nil && len(ref.Tags) > 0 { + for _, ta := range ref.Tags { + tags = tags + string(ta) + "," + } + tags = tags[:len(tags)-1] + } + lousrd.Tags = tags + if lousrdnum, err := o.Insert(&lousrd); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_reference success, "+ + "lousrdnum:", lousrdnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_reference failed, "+ + "lousrd:", lousrd, ", err: ", err) + o.Rollback() + return 0, err + } + } + } + lousv := OriginUpstreamVulType{CveId: num} + o.Delete(&lousv, "CveId") + osv.CveId = num + if osvnum, err := o.Insert(osv); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_vul_type success,"+ + " osvnum:", osvnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_vul_type failed, "+ + "osv:", osv, ", err: ", err) + o.Rollback() + return 0, err + } + ose.CveId = num + lousfs := OriginUpstreamFixSuggest{CveId: num} + errxxsx := o.Read(&lousfs, "CveId") + if errxxsx == orm.ErrNoRows || errxxsx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_fix_suggest, Data does not exist, cveId: ", num) + } else { + var lousfst []OriginUpstreamFixSuggestRef + louscNum, err := o.Raw("select * from cve_origin_upstream_fix_suggest_ref where fix_id = ?", + lousfs.FixId).QueryRows(&lousfst) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_fix_suggest_ref, Data does not exist, "+ + "louscNum: ", louscNum, "err: ", err) + } else { + for _, sc := range lousfst { + lorfrt := OriginUpstreamFixSuggestRefTag{FixRefId: sc.FixRefId} + o.Delete(&lorfrt, "FixRefId") + } + lo := OriginUpstreamFixSuggestRef{FixId: lousfs.FixId} + o.Delete(&lo, "FixId") + } + lousfsx := OriginUpstreamFixSuggest{CveId: num} + o.Delete(&lousfsx, "CveId") + } + osf.CveId = num + if osfnum, err := o.Insert(osf); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_fix_suggest success, "+ + "osfnum:", osfnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_fix_suggest failed, "+ + "osf:", osf, ", err: ", err) + o.Rollback() + return 0, err + } + if CveData.FixSuggest.References != nil && len(CveData.FixSuggest.References) > 0 { + for _, refer := range CveData.FixSuggest.References { + var lousfstr OriginUpstreamFixSuggestRef + lousfstr.FixId = osf.FixId + lousfstr.Name = refer.Name + lousfstr.Refsource = refer.Refsource + lousfstr.Url = refer.Url + if osfstrnum, err := o.Insert(&lousfstr); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref success, "+ + "osfstrnum:", osfstrnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref failed, "+ + "lousfstr:", lousfstr, ", err: ", err) + o.Rollback() + return 0, err + } + if refer.Tags != nil && len(refer.Tags) > 0 { + for _, refertag := range refer.Tags { + var loufsrtg OriginUpstreamFixSuggestRefTag + loufsrtg.FixRefId = lousfstr.FixRefId + loufsrtg.Name = refertag + if osfstgrnum, err := o.Insert(&loufsrtg); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref_tag success, "+ + "osfstgrnum:", osfstgrnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref_tag failed, "+ + "loufsrtg:", loufsrtg, ", err: ", err) + o.Rollback() + return 0, err + } + } + } + } + } + o.Commit() + } else { + logs.Info("update datas: ", ou) + ou.CveId = ouse.CveId + ou.Ids = ouse.Ids + ou.CreateTime = ouse.CreateTime + if num, err := o.Update(ou); err == nil { + logs.Info("CreateOriginCve, Update cve_origin_upstream success, num: ", num, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, Update cve_origin_upstream failed, ou:", ou, ", err: ", err) + o.Rollback() + return 0, err + } + num := ouse.CveId + od.CveId = num + lod := OriginUpstreamDesc{CveId: num} + o.Delete(&lod, "CveId") + if odnum, err := o.Insert(od); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_desc success, num:", odnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_desc failed, ou:", ou, ", err: ", err) + o.Rollback() + return 0, err + } + ous.CveId = num + lous := OriginUpstreamConfig{CveId: num} + err := o.Read(&lous, "CveId") + if err == orm.ErrNoRows || err == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_config, Data does not exist, cveId: ", num) + } else { + var lousc []OriginUpstreamConfigNode + louscNum, err := o.Raw("select * from cve_origin_upstream_config_node where conf_id = ?", + lous.ConfId).QueryRows(&lousc) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_config_node, Data does not exist,"+ + " louscNum: ", louscNum) + } else { + for _, lsc := range lousc { + ousnc := OriginUpstreamConfigNodeCpe{NodeId: lsc.NodeId} + o.Delete(&ousnc, "NodeId") + } + ousn := OriginUpstreamConfigNode{ConfId: lous.ConfId} + o.Delete(&ousn, "ConfId") + } + lousn := OriginUpstreamConfig{CveId: num} + o.Delete(&lousn, "CveId") + } + if ousnum, err := o.Insert(ous); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_config success, "+ + "ousnum:", ousnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_config failed, "+ + "ou:", ous, ", err: ", err) + o.Rollback() + return 0, err + } + if CveData.Configurations.Nodes != nil && len(CveData.Configurations.Nodes) > 0 { + for _, nodes := range CveData.Configurations.Nodes { + var Lnode OriginUpstreamConfigNode + Lnode.Operator = nodes.Operator + Lnode.ConfId = ous.ConfId + if lousnum, err := o.Insert(&Lnode); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_config_node success, "+ + "lousnum:", lousnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_config_node failed, "+ + "Lnode:", Lnode, ", err: ", err) + o.Rollback() + return 0, err + } + if nodes.Cpe != nil && len(nodes.Cpe) > 0 { + for _, nodCpe := range nodes.Cpe { + var ouscnc OriginUpstreamConfigNodeCpe + ouscnc.Cpe23Uri = nodCpe.Cpe23Uri + ouscnc.NodeId = Lnode.NodeId + ouscnc.CpeMatchString = nodCpe.CpeMatchString + ouscnc.Vulnerable = nodCpe.Vulnerable + if lousnumc, err := o.Insert(&ouscnc); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_config_node_cpe success, "+ + "lousnumc:", lousnumc, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_config_node_cpe failed, "+ + "ouscnc:", ouscnc, ", err: ", err) + o.Rollback() + return 0, err + } + } + } + } + } + osi.CveId = num + losi := OriginUpstreamImpact{CveId: num} + errxx := o.Read(&losi, "CveId") + if errxx == orm.ErrNoRows || errxx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_impact, Data does not exist, cveId: ", num) + } else { + var losis []OriginUpstreamImpactScore + losisNum, err := o.Raw("select * from cve_origin_upstream_impact_score where impact_id = ?", + losi.ImpactId).QueryRows(&losis) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_impact_score, Data does not exist,"+ + " losi.ImpactId: ", losi.ImpactId, ", err: ", err, ",losisNum: ", losisNum) + } else { + for _, sis := range losis { + if sis.CvssV3 == 1 && sis.BaseMetricV3 == 1 { + lousisv3 := OriginUpstreamImpactScoreV3{ScoreId: sis.ScoreId} + o.Delete(&lousisv3, "ScoreId") + } + if sis.CvssV2 == 1 && sis.BaseMetricV2 == 1 { + lousisv2 := OriginUpstreamImpactScoreV2{ScoreId: sis.ScoreId} + o.Delete(&lousisv2, "ScoreId") + } + } + losisx := OriginUpstreamImpactScore{ImpactId: losi.ImpactId} + o.Delete(&losisx, "ImpactId") + } + losix := OriginUpstreamImpact{CveId: num} + o.Delete(&losix, "CveId") + } + if losinum, err := o.Insert(osi); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact success, "+ + "lousnum:", losinum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact failed, "+ + "Lnode:", osi, ", err: ", err) + o.Rollback() + return 0, err + } + var lousist OriginUpstreamImpactScore + lousist.ImpactId = osi.ImpactId + lousist.BaseMetricV3 = 1 + lousist.BaseMetricV2 = 0 + lousist.CvssV3 = 1 + lousist.CvssV2 = 0 + lousist.Status = 1 + if lousistnum, err := o.Insert(&lousist); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ + "lousistnum:", lousistnum, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ + "lousist:", lousist, ", err: ", err) + o.Rollback() + return 0, err + } + var lousisv3 OriginUpstreamImpactScoreV3 + lousisv3.ScoreId = lousist.ScoreId + lousisv3.BaseScore = CveData.Impact.BaseMetricV3.CvssV3.BaseScore + lousisv3.VectorString = CveData.Impact.BaseMetricV3.CvssV3.VectorString + lousisv3.AttackComplexity = CveData.Impact.BaseMetricV3.CvssV3.AttackComplexity + lousisv3.AttackVector = CveData.Impact.BaseMetricV3.CvssV3.AttackVector + lousisv3.AvailabilityImpact = CveData.Impact.BaseMetricV3.CvssV3.AvailabilityImpact + lousisv3.BaseSeverity = CveData.Impact.BaseMetricV3.CvssV3.BaseSeverity + lousisv3.UserInteraction = CveData.Impact.BaseMetricV3.CvssV3.UserInteraction + lousisv3.PrivilegesRequired = CveData.Impact.BaseMetricV3.CvssV3.PrivilegesRequired + lousisv3.Version = CveData.Impact.BaseMetricV3.CvssV3.Version + lousisv3.ConfidentialityImpact = CveData.Impact.BaseMetricV3.CvssV3.ConfidentialityImpact + lousisv3.IntegrityImpact = CveData.Impact.BaseMetricV3.CvssV3.IntegrityImpact + lousisv3.Scope = CveData.Impact.BaseMetricV3.CvssV3.Scope + lousisv3.ImpactScore = CveData.Impact.BaseMetricV3.ImpactScore + lousisv3.ExploitabilityScore = CveData.Impact.BaseMetricV3.ExploitabilityScore + lousisv3.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV3.CvssV3.BaseScore) + if lousistnumv3, err := o.Insert(&lousisv3); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 success, "+ + "lousistnumv3:", lousistnumv3, ", cveNum", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 failed, "+ + "lousisv3:", lousisv3, ", err: ", err) + o.Rollback() + return 0, err + } + var lousistv2 OriginUpstreamImpactScore + lousistv2.ImpactId = osi.ImpactId + lousistv2.BaseMetricV3 = 0 + lousistv2.BaseMetricV2 = 1 + lousistv2.CvssV3 = 0 + lousistv2.CvssV2 = 1 + lousistv2.Status = 1 + if lousistnum, err := o.Insert(&lousistv2); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ + "lousistnum:", lousistnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ + "lousistv2:", lousistv2, ", err: ", err) + o.Rollback() + return 0, err + } + var lousisv2 OriginUpstreamImpactScoreV2 + lousisv2.ScoreId = lousistv2.ScoreId + lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore + lousisv2.AcInsufInfo = CveData.Impact.BaseMetricV2.AcInsufInfo + lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore + lousisv2.VectorString = CveData.Impact.BaseMetricV2.CvssV2.VectorString + lousisv2.AccessComplexity = CveData.Impact.BaseMetricV2.CvssV2.AccessComplexity + lousisv2.Authentication = CveData.Impact.BaseMetricV2.CvssV2.Authentication + lousisv2.AvailabilityImpact = CveData.Impact.BaseMetricV2.CvssV2.AvailabilityImpact + lousisv2.Version = CveData.Impact.BaseMetricV2.CvssV2.Version + lousisv2.ConfidentialityImpact = CveData.Impact.BaseMetricV2.CvssV2.ConfidentialityImpact + lousisv2.IntegrityImpact = CveData.Impact.BaseMetricV2.CvssV2.IntegrityImpact + lousisv2.AccessVector = CveData.Impact.BaseMetricV2.CvssV2.AccessVector + lousisv2.UserInteractionRequired = CveData.Impact.BaseMetricV2.UserInteractionRequired + lousisv2.Severity = CveData.Impact.BaseMetricV2.Severity + lousisv2.ObtainUserPrivilege = CveData.Impact.BaseMetricV2.ObtainUserPrivilege + lousisv2.ObtainAllPrivilege = CveData.Impact.BaseMetricV2.ObtainAllPrivilege + lousisv2.ObtainOtherPrivilege = CveData.Impact.BaseMetricV2.ObtainOtherPrivilege + lousisv2.ImpactScore = CveData.Impact.BaseMetricV2.ImpactScore + lousisv2.ExploitabilityScore = CveData.Impact.BaseMetricV2.ExploitabilityScore + lousisv2.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV2.CvssV2.BaseScore) + if lousistnumv2, err := o.Insert(&lousisv2); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 success, "+ + "lousistnumv2:", lousistnumv2, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 failed, "+ + "lousisv2:", lousisv2, ", err: ", err) + o.Rollback() + return 0, err + } + + osp.CveId = num + losp := OriginUpstreamPoc{CveId: num} + errxxx := o.Read(&losp, "CveId") + if errxxx == orm.ErrNoRows || errxxx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_impact, Data does not exist, cveId: ", num) + } else { + lospd := OriginUpstreamPoc{CveId: num} + o.Delete(&lospd, "CveId") + } + if ospnum, err := o.Insert(osp); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_poc success,"+ + " ospnum:", ospnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_poc failed, "+ + "osp:", osp, ", err: ", err) + o.Rollback() + return 0, err + } + ose.CveId = num + lose := OriginUpstreamEvent{CveId: num} + errx := o.Read(&lose, "CveId") + if errx == orm.ErrNoRows || errx == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_event, Data does not exist, cveId: ", num) + } else { + losed := OriginUpstreamEvent{CveId: num} + o.Delete(&losed, "CveId") + } + if osenum, err := o.Insert(ose); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_event success, "+ + "osenum:", osenum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_event failed, "+ + "ose:", ose, ", err: ", err) + o.Rollback() + return 0, err + } + lousr := OriginUpstreamReference{CveId: num} + o.Delete(&lousr, "CveId") + if CveData.ReferenceData != nil && len(CveData.ReferenceData) > 0 { + for _, ref := range CveData.ReferenceData { + var lousrd OriginUpstreamReference + lousrd.CveId = num + lousrd.Url = ref.Url + lousrd.Refsource = ref.Refsource + lousrd.Name = ref.Name + tags := "" + if ref.Tags != nil && len(ref.Tags) > 0 { + for _, ta := range ref.Tags { + tags = tags + string(ta) + "," + } + tags = tags[:len(tags)-1] + } + lousrd.Tags = tags + if lousrdnum, err := o.Insert(&lousrd); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_reference success, "+ + "lousrdnum:", lousrdnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_reference failed, "+ + "lousrd:", lousrd, ", err: ", err) + o.Rollback() + return 0, err + } + } + } + lousv := OriginUpstreamVulType{CveId: num} + o.Delete(&lousv, "CveId") + osv.CveId = num + if osvnum, err := o.Insert(osv); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_vul_type success, "+ + "osvnum:", osvnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_vul_type failed, "+ + "osv:", osv, ", err: ", err) + o.Rollback() + return 0, err + } + ose.CveId = num + lousfs := OriginUpstreamFixSuggest{CveId: num} + osErr := o.Read(&lousfs, "CveId") + if osErr == orm.ErrNoRows || osErr == orm.ErrMissPK { + logs.Info("CreateOriginCve, cve_origin_upstream_fix_suggest, Data does not exist, cveId: ", num) + } else { + var lousfst []OriginUpstreamFixSuggestRef + louscNum, err := o.Raw("select * from cve_origin_upstream_fix_suggest_ref where fix_id = ?", + lousfs.FixId).QueryRows(&lousfst) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_fix_suggest_ref, Data does not exist,"+ + " louscNum: ", louscNum, "err: ", err) + } else { + for _, sc := range lousfst { + lorfrt := OriginUpstreamFixSuggestRefTag{FixRefId: sc.FixRefId} + o.Delete(&lorfrt, "FixRefId") + } + lo := OriginUpstreamFixSuggestRef{FixId: lousfs.FixId} + o.Delete(&lo, "FixId") + } + lousfsx := OriginUpstreamFixSuggest{CveId: num} + o.Delete(&lousfsx, "CveId") + } + osf.CveId = num + if osfnum, err := o.Insert(osf); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_fix_suggest success, "+ + "osfnum:", osfnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_fix_suggest failed, "+ + "osf:", osf, ", err: ", err) + o.Rollback() + return 0, err + } + if CveData.FixSuggest.References != nil && len(CveData.FixSuggest.References) > 0 { + for _, refer := range CveData.FixSuggest.References { + var lousfstr OriginUpstreamFixSuggestRef + lousfstr.FixId = osf.FixId + lousfstr.Name = refer.Name + lousfstr.Refsource = refer.Refsource + lousfstr.Url = refer.Url + if osfstrnum, err := o.Insert(&lousfstr); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref success, "+ + "osfstrnum:", osfstrnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref failed, "+ + "lousfstr:", lousfstr, ", err: ", err) + o.Rollback() + return 0, err + } + if refer.Tags != nil && len(refer.Tags) > 0 { + for _, refertag := range refer.Tags { + var loufsrtg OriginUpstreamFixSuggestRefTag + loufsrtg.FixRefId = lousfstr.FixRefId + loufsrtg.Name = refertag + if osfstgrnum, err := o.Insert(&loufsrtg); err == nil { + logs.Info("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref_tag success, "+ + "osfstgrnum:", osfstgrnum, ", cveNum: ", ou.Ids) + } else { + logs.Error("CreateOriginCve, insert cve_origin_upstream_fix_suggest_ref_tag failed, "+ + "loufsrtg:", loufsrtg, ", err: ", err) + o.Rollback() + return 0, err + } + } + } + } + } + o.Commit() + } + } else { + logs.Error("CreateOriginCve, Transaction creation failed, cveNum:", ou.CveNum) + return 0, errs + } + return 0, nil +} + +func QueryCveErrorInfo(issueStatus int8, startDate, endDate string) ([]IssueTemplate, bool) { + o := orm.NewOrm() + var it []IssueTemplate + var num int64 + var err error + if startDate != "" { + num, err = o.Raw("select cve_num,error_description,create_time from cve_issue_template "+ + "where status > ? and create_time >= ? and create_time <= ?", + issueStatus, startDate, endDate).QueryRows(&it) + } else { + num, err = o.Raw("select cve_num,error_description,create_time from cve_issue_template "+ + "where status > ? and create_time <= ?", issueStatus, endDate).QueryRows(&it) + } + if err != nil || num == 0 { + logs.Info("QueryCveErrorInfo, ", issueStatus, startDate, endDate, + ", cve_issue_template, No corresponding data") + return it, false + } else { + logs.Info("QueryCveErrorInfo, ", issueStatus, startDate, endDate, + ", cve_issue_template, search successful") + return it, true + } +} + +func GetOpengaussYaml(opy *OpenGussYaml, colName ...string) error { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("openGauss request query parameters: ", *opy) + return errors.New("openGauss request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + err := o.Raw("select * from cve_open_guss_yaml where package_name = ? and version = ? "+ + "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) + if err != nil { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpengaussYaml1, err: ", err, *opy) + } + return err + } else { + err := o.Raw("select * from cve_open_guss_yaml where package_name = ? "+ + "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) + if err != nil { + logs.Error("GetOpengaussYaml2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return err + } +} + +func GetOpengaussYamlAll(opy *OpenGussYaml) (msy []OpenGussYaml, err error) { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("GetOpengaussYamlAll, openGauss request query parameters: ", *opy) + return msy, errors.New("openGauss request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + verStr := "(" + verList := strings.Split(opy.Version, ",") + for _, ver := range verList { + verStr += "'" + ver + "'," + } + verStr = verStr[:len(verStr)-1] + ")" + sql := fmt.Sprintf(`select * from cve_open_guss_yaml where package_name = '%s' +and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) + num, err := o.Raw(sql).QueryRows(&msy) + if num == 0 { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpengaussYamlAll, err: ", err, *opy) + } + return msy, err + } else { + num, err := o.Raw("select * from cve_open_guss_yaml where package_name = ? "+ + "order by id asc", opy.PackageName).QueryRows(&msy) + if num == 0 { + logs.Error("GetOpengaussYamlAll2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return msy, err + } +} + +func GetMindSporeYaml(opy *MindSporeYaml, colName ...string) error { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("Mindspore request query parameters: ", *opy) + return errors.New("Mindspore request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? and version = ? "+ + "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) + if err != nil { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetMindSporeYaml1, err: ", err, *opy) + } + return err + } else { + err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? "+ + "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) + if err != nil { + logs.Error("GetMindSporeYaml2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return err + } +} + +func GetMindSporeYamlAll(opy *MindSporeYaml) (msy []MindSporeYaml, err error) { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("GetMindSporeYamlAll, Mindspore request query parameters: ", *opy) + return msy, errors.New("Mindspore request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + verStr := "(" + verList := strings.Split(opy.Version, ",") + for _, ver := range verList { + verStr += "'" + ver + "'," + } + verStr = verStr[:len(verStr)-1] + ")" + sql := fmt.Sprintf(`select * from cve_mind_spore_yaml where package_name = '%s' +and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) + num, err := o.Raw(sql).QueryRows(&msy) + if num == 0 { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetMindSporeYamlAll, err: ", err, *opy) + } + return msy, err + } else { + num, err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? "+ + "order by id asc", opy.PackageName).QueryRows(&msy) + if num == 0 { + logs.Error("GetMindSporeYamlAll2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return msy, err + } +} + +func InsertOriginCveRecord(our *OriginUpstreamRecord) error { + o := orm.NewOrm() + id, err := o.Insert(our) + logs.Info("InsertOriginCveRecord, id: ", id, ", err: ", err) + return err +} + +func DeleteOriginCveRecord(beforeDate string) { + o := orm.NewOrm() + err := o.Raw("delete from cve_origin_upstream_record where create_time < ?", beforeDate).QueryRow() + logs.Info("DeleteOriginCveRecord", err) +} + +func GetOpenLookengYaml(opy *OpenLookengYaml, colName ...string) error { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("OpenLooKeng request query parameters: ", *opy) + return errors.New("OpenLooKeng request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? and version = ? "+ + "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) + if err != nil { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpenLookengYaml1, err: ", err, *opy) + } + return err + } else { + err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? "+ + "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) + if err != nil { + logs.Error("GetOpenLookengYaml12, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return err + } +} + +func GetOpenLookengYamlAll(opy *OpenLookengYaml) (msy []OpenLookengYaml, err error) { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("GetOpenLookengYamlAll, openLookeng request query parameters: ", *opy) + return msy, errors.New("openLookeng request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + verStr := "(" + verList := strings.Split(opy.Version, ",") + for _, ver := range verList { + verStr += "'" + ver + "'," + } + verStr = verStr[:len(verStr)-1] + ")" + sql := fmt.Sprintf(`select * from cve_open_lookeng_yaml where package_name = '%s' +and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) + num, err := o.Raw(sql).QueryRows(&msy) + if num == 0 { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpenLookengYamlAll, err: ", err, *opy) + } + return msy, err + } else { + num, err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? "+ + "order by id asc", opy.PackageName).QueryRows(&msy) + if num == 0 { + logs.Error("GetOpenLookengYamlAll2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return msy, err + } +} diff --git a/cve-vulner-manager/models/user.go b/cve-vulner-manager/models/user.go new file mode 100644 index 0000000000000000000000000000000000000000..3952c048c0f68e6602cf9882f3abcda74afd5fd9 --- /dev/null +++ b/cve-vulner-manager/models/user.go @@ -0,0 +1,87 @@ +package models + +import ( + "errors" + "strconv" + "time" +) + +var ( + UserList map[string]*User +) + +func init() { + UserList = make(map[string]*User) + u := User{"user_11111", "astaxie", "11111", + Profile{"male", 20, "Singapore", "astaxie@gmail.com"}} + UserList["user_11111"] = &u +} + +type User struct { + Id string + Username string + Password string + Profile Profile +} + +type Profile struct { + Gender string + Age int + Address string + Email string +} + +func AddUser(u User) string { + u.Id = "user_" + strconv.FormatInt(time.Now().UnixNano(), 10) + UserList[u.Id] = &u + return u.Id +} + +func GetUser(uid string) (u *User, err error) { + if u, ok := UserList[uid]; ok { + return u, nil + } + return nil, errors.New(" User not exists") +} + +func GetAllUsers() map[string]*User { + return UserList +} + +func UpdateUser(uid string, uu *User) (a *User, err error) { + if u, ok := UserList[uid]; ok { + if uu.Username != "" { + u.Username = uu.Username + } + if uu.Password != "" { + u.Password = uu.Password + } + if uu.Profile.Age != 0 { + u.Profile.Age = uu.Profile.Age + } + if uu.Profile.Address != "" { + u.Profile.Address = uu.Profile.Address + } + if uu.Profile.Gender != "" { + u.Profile.Gender = uu.Profile.Gender + } + if uu.Profile.Email != "" { + u.Profile.Email = uu.Profile.Email + } + return u, nil + } + return nil, errors.New(" User Not Exist") +} + +func Login(username, password string) bool { + for _, u := range UserList { + if u.Username == username && u.Password == password { + return true + } + } + return false +} + +func DeleteUser(uid string) { + delete(UserList, uid) +} diff --git a/cve-vulner-manager/models/ymal.go b/cve-vulner-manager/models/ymal.go new file mode 100644 index 0000000000000000000000000000000000000000..1b9bca127f402cdc601b63b04d5f5448ab345a9b --- /dev/null +++ b/cve-vulner-manager/models/ymal.go @@ -0,0 +1,339 @@ +package models + +import ( + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) + +func GetYamlTable(gt *[]GitPackageTable) (int64, error) { + o := orm.NewOrm() + num, errx := o.Raw("select *" + + " from cve_git_package_table").QueryRows(gt) + if errx != nil { + logs.Error("GetYamlTable, Get data does not exist, err: ", errx) + } + return num, errx +} + +func GetCpePackName(pc *PackageCpe, packName string) (error) { + o := orm.NewOrm() + errx := o.Raw("select id, cpe_packname"+ + " from cve_package_cpe where packname = ?", packName).QueryRow(pc) + return errx +} + +func CreateYamlTable(gt *GitPackageTable) (TableId int64, err error) { + o := orm.NewOrm() + var localgt GitPackageTable + if gt.TableName == "openEuler_LTS_20.03" { + gt.Brands = "openEuler-20.03-LTS" + } else { + gt.Brands = "master" + } + errx := o.Raw("select *"+ + " from cve_git_package_table where table_name = ? ", gt.TableName).QueryRow(&localgt) + if errx != nil { + var TableId int64 + if TableId, err = o.Insert(gt); err == nil { + logs.Info("CreateYamlTable, insert cve_git_package_table success, "+ + "TableId: ", TableId, ", tableName: ", gt.TableName) + } else { + logs.Error("CreateYamlTable, insert cve_git_package_table failed,"+ + " tableName:", gt.TableName, ", err: ", err) + return 0, err + } + return TableId, nil + } else { + gt.TableId = localgt.TableId + gt.Brands = localgt.Brands + if num, err := o.Update(gt); err == nil { + logs.Info("CreateYamlTable, update cve_git_package_table success, "+ + "num: ", num, ", TableName: ", gt.TableName) + } else { + logs.Error("CreateYamlTable, update cve_git_package_table failed, "+ + "TableName:", gt.TableName, ", err: ", err) + return 0, err + } + return gt.TableId, nil + } +} + +func GetSingleYaml(ge *GitOpenEuler) (bool, error) { + o := orm.NewOrm() + if ge.Version != "" && ge.Release != "" { + localge := GitOpenEuler{PackageName: ge.PackageName, Version: ge.Version, Status: 1} + err := o.Read(&localge, "PackageName", "Version", "Release") + logs.Info("GetSingleYaml1, Query yaml data results: ", ge, + ", localge: ", localge, ", err: ", err) + if err == orm.ErrNoRows { + return false, err + } else if err == orm.ErrMissPK { + return false, err + } else { + return true, err + } + } else { + localge := GitOpenEuler{PackageName: ge.PackageName, Status: 1} + err := o.Read(&localge, "PackageName") + logs.Info("GetSingleYaml2, Query yaml data results: ", ge, + ", localge: ", localge, ", err: ", err) + if err == orm.ErrNoRows { + return false, err + } else if err == orm.ErrMissPK { + return false, err + } else { + return true, err + } + } +} + +func InsertOpenEulerRelate(getr *GitOpenEulerTableRelate) (id int64, typex string, err error) { + o := orm.NewOrm() + var localge GitOpenEulerTableRelate + errx := o.Raw("select *"+ + " from cve_git_open_euler_table_relate where git_id = ? and table_name = ?", + getr.GitId, getr.TableName).QueryRow(&localge) + if errx != nil { + var relateId int64 + if relateId, err = o.Insert(getr); err == nil { + logs.Info("InsertOpenEulerRelate, insert cve_git_open_euler_table_relate success, "+ + "RelateId: ", relateId, "tableName: ", getr.TableName) + } else { + logs.Error("InsertOpenEulerRelate, insert cve_git_open_euler_table_relate failed, "+ + "tableName: ", getr.TableName, ", err: ", err) + return 0, "insert", err + } + return relateId, "insert", nil + } else { + getr.RelateId = localge.RelateId + if _, err := o.Update(getr); err == nil { + logs.Info("InsertOpenEulerRelate, update cve_git_open_euler_table_relate success, "+ + "RelateId: ", getr.RelateId, "tableName: ", getr.TableName) + } else { + logs.Error("InsertOpenEulerRelate, update cve_git_open_euler_table_relate failed, "+ + "tableName: ", getr.TableName, ", err: ", err) + return getr.RelateId, "update", err + } + return getr.RelateId, "update", nil + } +} + +func CreateYaml(ge *GitOpenEuler, tableValue GitPackageTable) (id int64, typex string, err error) { + o := orm.NewOrm() + var localge GitOpenEuler + var getr GitOpenEulerTableRelate + getr.TableName = tableValue.TableName + getr.Brands = tableValue.Brands + errx := o.Raw("select *"+ + " from cve_git_open_euler where package_name = ? and version = ?", + ge.PackageName, ge.Version).QueryRow(&localge) + if errx != nil { + logs.Info("createYaml: ", localge, ",errx: ", errx) + ge.Status = 1 + var GitId int64 + if GitId, err = o.Insert(ge); err == nil { + logs.Info("CreateYaml, insert cve_git_open_euler success, GitId: ", + GitId, ", packName: ", ge.PackageName) + } else { + logs.Error("CreateYaml, insert cve_git_open_euler failed, "+ + "packName: ", ge.PackageName, ", err: ", err) + return 0, "insert", err + } + getr.GitId = GitId + InsertOpenEulerRelate(&getr) + return GitId, "insert", nil + } else { + logs.Info("createYaml: ", localge, "GitOpenEuler: ", ge) + ge.Status = 1 + ge.GitId = localge.GitId + if _, err := o.Update(ge); err == nil { + logs.Info("CreateYaml, update cve_git_open_euler success, "+ + "GitId: ", ge.GitId, ", packName: ", ge.PackageName) + } else { + logs.Error("CreateYaml, update cve_git_open_euler failed, "+ + "packName: ", ge.PackageName, ", err: ", err) + return ge.GitId, "update", err + } + getr.GitId = ge.GitId + InsertOpenEulerRelate(&getr) + return ge.GitId, "update", nil + } +} + +func CreateYamlDetail(gp *GitPackageInfo, ge GitOpenEuler) (id int64, typex string, err error) { + o := orm.NewOrm() + var localgp GitPackageInfo + errx := o.Raw("select *"+ + " from cve_git_package_info where git_id = ? and package_name = ? and version = ?", + ge.GitId, gp.PackageName, gp.Version).QueryRow(&localgp) + if errx != nil { + gp.Status = 0 + var DetailId int64 + if DetailId, err = o.Insert(gp); err == nil { + logs.Info("CreateYamlDetail, insert cve_git_package_info success, "+ + "DetailId: ", DetailId, ",PackageName: ", gp.PackageName) + } else { + logs.Error("CreateYamlDetail, insert cve_git_package_info failed, "+ + "PackageName: ", gp.PackageName, ",err: ", err) + return 0, "insert", err + } + return DetailId, "insert", nil + } else { + gp.DetailId = localgp.DetailId + if _, err := o.Update(gp); err == nil { + logs.Info("CreateYamlDetail, update cve_git_package_info success, "+ + "DetailId: ", gp.DetailId, ",packName: ", gp.PackageName) + } else { + logs.Error("CreateYamlDetail, update cve_git_package_info failed, "+ + "packName: ", gp.PackageName, ",err: ", err) + return gp.DetailId, "update", err + } + return gp.DetailId, "update", nil + } +} + +func CreateYamlSubPack(gb *GitSubPack) (SubId int64, typex string, err error) { + o := orm.NewOrm() + var localgb GitSubPack + errx := o.Raw("select *"+ + " from cve_git_sub_pack where detail_id = ? and ids = ? and sub_pack_name = ?", + gb.DetailId, gb.Ids, gb.SubPackName).QueryRow(&localgb) + if errx != nil { + var SubId int64 + if SubId, err = o.Insert(gb); err == nil { + logs.Info("CreateYamlSubPack, insert cve_git_sub_pack success,"+ + " SubId: ", SubId, ",SubPackName: ", gb.SubPackName) + } else { + logs.Error("CreateYamlSubPack, insert cve_git_sub_pack failed, "+ + "SubPackName: ", gb.SubPackName, ",err: ", err) + return 0, "insert", err + } + return SubId, "insert", nil + } + return localgb.SubId, "update", errx +} + +func CreateYamlSubPackProvides(gs *GitSubPackProvides) (ProvideId int64, typex string, err error) { + o := orm.NewOrm() + var localgs GitSubPackProvides + errx := o.Raw("select *"+ + " from cve_git_sub_pack_provides where sub_id = ? and ids = ? and provide_name = ?", + gs.SubId, gs.Ids, gs.ProvideName).QueryRow(&localgs) + if errx != nil { + var ProvideId int64 + if ProvideId, err = o.Insert(gs); err == nil { + logs.Info("CreateYamlSubPack, insert cve_git_sub_pack_provides success, "+ + "ProvideId: ", ProvideId, ", ProvideName: ", gs.ProvideName) + } else { + logs.Error("CreateYamlSubPack, insert cve_git_sub_pack_provides failed, "+ + "ProvideName: ", gs.ProvideName, ",err: ", err) + return 0, "insert", err + } + return ProvideId, "insert", nil + } + return localgs.ProvideId, "update", errx +} + +func CreateYamlSubPackRequiredb(gs *GitSubPackRequiredby) (Id int64, typex string, err error) { + o := orm.NewOrm() + var localgs GitSubPackRequiredby + errx := o.Raw("select *"+ + " from cve_git_sub_pack_requiredby where provide_id = ? and requiredb = ?", + gs.ProvideId, gs.Requiredby).QueryRow(&localgs) + if errx != nil { + var Id int64 + if Id, err = o.Insert(gs); err == nil { + logs.Info("CreateYamlSubPackRequiredb, insert cve_git_sub_pack_requiredby success, "+ + "ID: ", Id, ", requiredb: ", gs.Requiredby) + } else { + logs.Error("CreateYamlSubPackRequiredb, insert cve_git_sub_pack_requiredby failed, "+ + "requiredb: ", gs.Requiredby, ", err: ", err) + return 0, "insert", err + } + return Id, "insert", nil + } + return localgs.Id, "update", errx +} + +func CreateYamlSubPackRequires(gs *GitSubPackRequire) (RequireId int64, typex string, err error) { + o := orm.NewOrm() + var localgs GitSubPackRequire + errx := o.Raw("select *"+ + " from cve_git_sub_pack_require where sub_id = ? and ids = ? and require_name = ?", + gs.SubId, gs.Ids, gs.RequireName).QueryRow(&localgs) + if errx != nil { + var RequireId int64 + if RequireId, err = o.Insert(gs); err == nil { + logs.Info("CreateYamlSubPackRequires, insert cve_git_sub_pack_require success,"+ + " RequireId: ", RequireId, ",RequireName: ", gs.RequireName) + } else { + logs.Error("CreateYamlSubPackRequires, insert cve_git_sub_pack_require failed, "+ + "RequireName: ", gs.RequireName, ", err: ", err) + return 0, "insert", err + } + return RequireId, "insert", nil + } + return localgs.RequireId, "update", errx +} + +func QueryEulerRepoOrigin(ogk *OpenEulerRepoOrigin, field ...string) error { + o := orm.NewOrm() + err := o.Read(ogk, field...) + return err +} + +func QueryEulerRepoOriginByNmae(packageName string) (oer []OpenEulerRepoOrigin) { + o := orm.NewOrm() + num, geErr := o.Raw("select *"+ + " from cve_open_euler_repo_origin where package_name = ?", packageName).QueryRows(&oer) + if num == 0 { + logs.Error("QueryEulerRepoOriginByNmae, Get data does not exist, geErr: ", geErr) + } + return +} + +// insert data +func InsertEulerRepoOrigin(ogk *OpenEulerRepoOrigin) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(ogk) + return id, err +} + +func UpdateEulerRepoOrigin(ogk *OpenEulerRepoOrigin, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogk, fields...) + return err +} + +func GetEulerYamlList(yamlId int64, count int) (oer []OpenEulerRepoOrigin) { + o := orm.NewOrm() + num, geErr := o.Raw("select *"+ + " from cve_open_euler_repo_origin where id > ? order by id asc limit ?", yamlId, count).QueryRows(&oer) + if num == 0 { + logs.Error("GetEulerYamlList, Get data does not exist, geErr: ", geErr) + } + return +} + +func GetGitOpenEulerList(gitId int64, count int) (goe []GitOpenEuler) { + o := orm.NewOrm() + num, geErr := o.Raw("select *"+ + " from cve_git_open_euler where git_id > ? and package_id < ? and status = 1 order by git_id asc limit ?", + gitId, 10000000, count).QueryRows(&goe) + if num == 0 { + logs.Error("GetGitOpenEulerList, Get data does not exist, geErr: ", geErr) + } + return +} + +func DelOpenEulerRepoOriginByName(packageName string) { + o := orm.NewOrm() + _ = o.Raw("DELETE FROM cve_open_euler_repo_origin WHERE package_name = ?", packageName).QueryRow() + return +} + +func TruncateOpenEulerRepoOrigin() { + o := orm.NewOrm() + _ = o.Raw("TRUNCATE table cve_open_euler_repo_origin").QueryRow() + return +} diff --git a/cve-vulner-manager/obs/auth.go b/cve-vulner-manager/obs/auth.go new file mode 100644 index 0000000000000000000000000000000000000000..66b4a973c1375643d4f6a2783d0e05f65c9da655 --- /dev/null +++ b/cve-vulner-manager/obs/auth.go @@ -0,0 +1,468 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "net/url" + "sort" + "strings" + "time" +) + +func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, expires int64) (requestURL string, err error) { + sh := obsClient.getSecurity() + isAkSkEmpty := sh.ak == "" || sh.sk == "" + if isAkSkEmpty == false && sh.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = sh.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = sh.securityToken + } + } + requestURL, canonicalizedURL := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestURL, err := url.Parse(requestURL) + if err != nil { + return "", err + } + encodeHeaders(headers) + hostName := parsedRequestURL.Host + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if isAkSkEmpty { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + if isV4 { + date, parseDateErr := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } + delete(headers, HEADER_DATE_CAMEL) + shortDate := date.Format(SHORT_DATE_FORMAT) + longDate := date.Format(LONG_DATE_FORMAT) + if len(headers[HEADER_HOST_CAMEL]) != 0 { + index := strings.LastIndex(headers[HEADER_HOST_CAMEL][0], ":") + if index != -1 { + port := headers[HEADER_HOST_CAMEL][0][index+1:] + if port == "80" || port == "443" { + headers[HEADER_HOST_CAMEL] = []string{headers[HEADER_HOST_CAMEL][0][:index]} + } + } + + } + + signedHeaders, _headers := getSignedHeaders(headers) + + credential, scope := getCredential(sh.ak, obsClient.conf.region, shortDate) + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate + params[PARAM_EXPIRES_AMZ_CAMEL] = Int64ToString(expires) + params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = strings.Join(signedHeaders, ";") + + requestURL, canonicalizedURL = obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestURL, _err := url.Parse(requestURL) + if _err != nil { + return "", _err + } + + stringToSign := getV4StringToSign(method, canonicalizedURL, parsedRequestURL.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, signedHeaders, _headers) + signature := getSignature(stringToSign, sh.sk, obsClient.conf.region, shortDate) + + requestURL += fmt.Sprintf("&%s=%s", PARAM_SIGNATURE_AMZ_CAMEL, UrlEncode(signature, false)) + + } else { + originDate := headers[HEADER_DATE_CAMEL][0] + date, parseDateErr := time.Parse(RFC1123_FORMAT, originDate) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } + expires += date.Unix() + headers[HEADER_DATE_CAMEL] = []string{Int64ToString(expires)} + + stringToSign := getV2StringToSign(method, canonicalizedURL, headers, obsClient.conf.signature == SignatureObs) + signature := UrlEncode(Base64Encode(HmacSha1([]byte(sh.sk), []byte(stringToSign))), false) + if strings.Index(requestURL, "?") < 0 { + requestURL += "?" + } else { + requestURL += "&" + } + delete(headers, HEADER_DATE_CAMEL) + + if obsClient.conf.signature != SignatureObs { + requestURL += "AWS" + } + requestURL += fmt.Sprintf("AccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(sh.ak, false), expires, signature) + } + } + + return +} + +func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, hostName string) (requestURL string, err error) { + sh := obsClient.getSecurity() + isAkSkEmpty := sh.ak == "" || sh.sk == "" + if isAkSkEmpty == false && sh.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + headers[HEADER_STS_TOKEN_OBS] = []string{sh.securityToken} + } else { + headers[HEADER_STS_TOKEN_AMZ] = []string{sh.securityToken} + } + } + isObs := obsClient.conf.signature == SignatureObs + requestURL, canonicalizedURL := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestURL, err := url.Parse(requestURL) + if err != nil { + return "", err + } + encodeHeaders(headers) + + if hostName == "" { + hostName = parsedRequestURL.Host + } + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if isAkSkEmpty { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + ak := sh.ak + sk := sh.sk + var authorization string + if isV4 { + headers[HEADER_CONTENT_SHA256_AMZ] = []string{UNSIGNED_PAYLOAD} + ret := v4Auth(ak, sk, obsClient.conf.region, method, canonicalizedURL, parsedRequestURL.RawQuery, headers) + authorization = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else { + ret := v2Auth(ak, sk, method, canonicalizedURL, headers, isObs) + hashPrefix := V2_HASH_PREFIX + if isObs { + hashPrefix = OBS_HASH_PREFIX + } + authorization = fmt.Sprintf("%s %s:%s", hashPrefix, ak, ret["Signature"]) + } + headers[HEADER_AUTH_CAMEL] = []string{authorization} + } + return +} + +func prepareHostAndDate(headers map[string][]string, hostName string, isV4 bool) { + headers[HEADER_HOST_CAMEL] = []string{hostName} + if date, ok := headers[HEADER_DATE_AMZ]; ok { + flag := false + if len(date) == 1 { + if isV4 { + if t, err := time.Parse(LONG_DATE_FORMAT, date[0]); err == nil { + headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(t)} + flag = true + } + } else { + if strings.HasSuffix(date[0], "GMT") { + headers[HEADER_DATE_CAMEL] = []string{date[0]} + flag = true + } + } + } + if !flag { + delete(headers, HEADER_DATE_AMZ) + } + } + if _, ok := headers[HEADER_DATE_CAMEL]; !ok { + headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(time.Now().UTC())} + } +} + +func encodeHeaders(headers map[string][]string) { + for key, values := range headers { + for index, value := range values { + values[index] = UrlEncode(value, true) + } + headers[key] = values + } +} + +func attachHeaders(headers map[string][]string, isObs bool) string { + length := len(headers) + _headers := make(map[string][]string, length) + keys := make([]string, 0, length) + + for key, value := range headers { + _key := strings.ToLower(strings.TrimSpace(key)) + if _key != "" { + prefixheader := HEADER_PREFIX + if isObs { + prefixheader = HEADER_PREFIX_OBS + } + if _key == "content-md5" || _key == "content-type" || _key == "date" || strings.HasPrefix(_key, prefixheader) { + keys = append(keys, _key) + _headers[_key] = value + } + } else { + delete(headers, key) + } + } + + for _, interestedHeader := range interestedHeaders { + if _, ok := _headers[interestedHeader]; !ok { + _headers[interestedHeader] = []string{""} + keys = append(keys, interestedHeader) + } + } + dateCamelHeader := PARAM_DATE_AMZ_CAMEL + dataHeader := HEADER_DATE_AMZ + if isObs { + dateCamelHeader = PARAM_DATE_OBS_CAMEL + dataHeader = HEADER_DATE_OBS + } + if _, ok := _headers[HEADER_DATE_CAMEL]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } else if _, ok := _headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } + + sort.Strings(keys) + + stringToSign := make([]string, 0, len(keys)) + for _, key := range keys { + var value string + prefixHeader := HEADER_PREFIX + prefixMetaHeader := HEADER_PREFIX_META + if isObs { + prefixHeader = HEADER_PREFIX_OBS + prefixMetaHeader = HEADER_PREFIX_META_OBS + } + if strings.HasPrefix(key, prefixHeader) { + if strings.HasPrefix(key, prefixMetaHeader) { + for index, v := range _headers[key] { + value += strings.TrimSpace(v) + if index != len(_headers[key])-1 { + value += "," + } + } + } else { + value = strings.Join(_headers[key], ",") + } + value = fmt.Sprintf("%s:%s", key, value) + } else { + value = strings.Join(_headers[key], ",") + } + stringToSign = append(stringToSign, value) + } + return strings.Join(stringToSign, "\n") +} + +func getV2StringToSign(method, canonicalizedURL string, headers map[string][]string, isObs bool) string { + stringToSign := strings.Join([]string{method, "\n", attachHeaders(headers, isObs), "\n", canonicalizedURL}, "") + + var isSecurityToken bool + var securityToken []string + if isObs { + securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_OBS] + } else { + securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_AMZ] + } + var query []string + if !isSecurityToken { + parmas := strings.Split(canonicalizedURL, "?") + if len(parmas) > 1 { + query = strings.Split(parmas[1], "&") + for _, value := range query { + if strings.HasPrefix(value, HEADER_STS_TOKEN_AMZ+"=") || strings.HasPrefix(value, HEADER_STS_TOKEN_OBS+"=") { + if value[len(HEADER_STS_TOKEN_AMZ)+1:] != "" { + securityToken = []string{value[len(HEADER_STS_TOKEN_AMZ)+1:]} + isSecurityToken = true + } + } + } + } + } + logStringToSign := stringToSign + if isSecurityToken && len(securityToken) > 0 { + logStringToSign = strings.Replace(logStringToSign, securityToken[0], "******", -1) + } + doLog(LEVEL_DEBUG, "The v2 auth stringToSign:\n%s", logStringToSign) + return stringToSign +} + +func v2Auth(ak, sk, method, canonicalizedURL string, headers map[string][]string, isObs bool) map[string]string { + stringToSign := getV2StringToSign(method, canonicalizedURL, headers, isObs) + return map[string]string{"Signature": Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign)))} +} + +func getScope(region, shortDate string) string { + return fmt.Sprintf("%s/%s/%s/%s", shortDate, region, V4_SERVICE_NAME, V4_SERVICE_SUFFIX) +} + +func getCredential(ak, region, shortDate string) (string, string) { + scope := getScope(region, shortDate) + return fmt.Sprintf("%s/%s", ak, scope), scope +} + +func getV4StringToSign(method, canonicalizedURL, queryURL, scope, longDate, payload string, signedHeaders []string, headers map[string][]string) string { + canonicalRequest := make([]string, 0, 10+len(signedHeaders)*4) + canonicalRequest = append(canonicalRequest, method) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, canonicalizedURL) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, queryURL) + canonicalRequest = append(canonicalRequest, "\n") + + for _, signedHeader := range signedHeaders { + values, _ := headers[signedHeader] + for _, value := range values { + canonicalRequest = append(canonicalRequest, signedHeader) + canonicalRequest = append(canonicalRequest, ":") + canonicalRequest = append(canonicalRequest, value) + canonicalRequest = append(canonicalRequest, "\n") + } + } + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, strings.Join(signedHeaders, ";")) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, payload) + + _canonicalRequest := strings.Join(canonicalRequest, "") + + var isSecurityToken bool + var securityToken []string + if securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_OBS]; !isSecurityToken { + securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_AMZ] + } + var query []string + if !isSecurityToken { + query = strings.Split(queryURL, "&") + for _, value := range query { + if strings.HasPrefix(value, HEADER_STS_TOKEN_AMZ+"=") || strings.HasPrefix(value, HEADER_STS_TOKEN_OBS+"=") { + if value[len(HEADER_STS_TOKEN_AMZ)+1:] != "" { + securityToken = []string{value[len(HEADER_STS_TOKEN_AMZ)+1:]} + isSecurityToken = true + } + } + } + } + logCanonicalRequest := _canonicalRequest + if isSecurityToken && len(securityToken) > 0 { + logCanonicalRequest = strings.Replace(logCanonicalRequest, securityToken[0], "******", -1) + } + doLog(LEVEL_DEBUG, "The v4 auth canonicalRequest:\n%s", logCanonicalRequest) + + stringToSign := make([]string, 0, 7) + stringToSign = append(stringToSign, V4_HASH_PREFIX) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, longDate) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, scope) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, HexSha256([]byte(_canonicalRequest))) + + _stringToSign := strings.Join(stringToSign, "") + + doLog(LEVEL_DEBUG, "The v4 auth stringToSign:\n%s", _stringToSign) + return _stringToSign +} + +func getSignedHeaders(headers map[string][]string) ([]string, map[string][]string) { + length := len(headers) + _headers := make(map[string][]string, length) + signedHeaders := make([]string, 0, length) + for key, value := range headers { + _key := strings.ToLower(strings.TrimSpace(key)) + if _key != "" { + signedHeaders = append(signedHeaders, _key) + _headers[_key] = value + } else { + delete(headers, key) + } + } + sort.Strings(signedHeaders) + return signedHeaders, _headers +} + +func getSignature(stringToSign, sk, region, shortDate string) string { + key := HmacSha256([]byte(V4_HASH_PRE+sk), []byte(shortDate)) + key = HmacSha256(key, []byte(region)) + key = HmacSha256(key, []byte(V4_SERVICE_NAME)) + key = HmacSha256(key, []byte(V4_SERVICE_SUFFIX)) + return Hex(HmacSha256(key, []byte(stringToSign))) +} + +// V4Auth is a wrapper for v4Auth +func V4Auth(ak, sk, region, method, canonicalizedURL, queryURL string, headers map[string][]string) map[string]string { + return v4Auth(ak, sk, region, method, canonicalizedURL, queryURL, headers) +} + +func v4Auth(ak, sk, region, method, canonicalizedURL, queryURL string, headers map[string][]string) map[string]string { + var t time.Time + if val, ok := headers[HEADER_DATE_AMZ]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[HEADER_DATE_CAMEL]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else { + t = time.Now().UTC() + } + shortDate := t.Format(SHORT_DATE_FORMAT) + longDate := t.Format(LONG_DATE_FORMAT) + + signedHeaders, _headers := getSignedHeaders(headers) + + credential, scope := getCredential(ak, region, shortDate) + + payload := UNSIGNED_PAYLOAD + if val, ok := headers[HEADER_CONTENT_SHA256_AMZ]; ok { + payload = val[0] + } + stringToSign := getV4StringToSign(method, canonicalizedURL, queryURL, scope, longDate, payload, signedHeaders, _headers) + + signature := getSignature(stringToSign, sk, region, shortDate) + + ret := make(map[string]string, 3) + ret["Credential"] = credential + ret["SignedHeaders"] = strings.Join(signedHeaders, ";") + ret["Signature"] = signature + return ret +} diff --git a/cve-vulner-manager/obs/client.go b/cve-vulner-manager/obs/client.go new file mode 100644 index 0000000000000000000000000000000000000000..cbff2c1434186cb8221eafcd7f71d50a9ed9b1ff --- /dev/null +++ b/cve-vulner-manager/obs/client.go @@ -0,0 +1,1328 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strings" +) + +// ObsClient defines OBS client. +type ObsClient struct { + conf *config + httpClient *http.Client +} + +// New creates a new ObsClient instance. +func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) { + conf := &config{endpoint: endpoint} + conf.securityProviders = make([]securityProvider, 0, 3) + conf.securityProviders = append(conf.securityProviders, NewBasicSecurityProvider(ak, sk, "")) + + conf.maxRetryCount = -1 + conf.maxRedirectCount = -1 + for _, configurer := range configurers { + configurer(conf) + } + + if err := conf.initConfigWithDefault(); err != nil { + return nil, err + } + err := conf.getTransport() + if err != nil { + return nil, err + } + + if isWarnLogEnabled() { + info := make([]string, 3) + info[0] = fmt.Sprintf("[OBS SDK Version=%s", obsSdkVersion) + info[1] = fmt.Sprintf("Endpoint=%s", conf.endpoint) + accessMode := "Virtual Hosting" + if conf.pathStyle { + accessMode = "Path" + } + info[2] = fmt.Sprintf("Access Mode=%s]", accessMode) + doLog(LEVEL_WARN, strings.Join(info, "];[")) + } + doLog(LEVEL_DEBUG, "Create obsclient with config:\n%s\n", conf) + obsClient := &ObsClient{conf: conf, httpClient: &http.Client{Transport: conf.transport, CheckRedirect: checkRedirectFunc}} + return obsClient, nil +} + +// Refresh refreshes ak, sk and securityToken for obsClient. +func (obsClient ObsClient) Refresh(ak, sk, securityToken string) { + for _, sp := range obsClient.conf.securityProviders { + if bsp, ok := sp.(*BasicSecurityProvider); ok { + bsp.refresh(strings.TrimSpace(ak), strings.TrimSpace(sk), strings.TrimSpace(securityToken)) + break + } + } +} + +func (obsClient ObsClient) getSecurity() securityHolder { + if obsClient.conf.securityProviders != nil { + for _, sp := range obsClient.conf.securityProviders { + if sp == nil { + continue + } + sh := sp.getSecurity() + if sh.ak != "" && sh.sk != "" { + return sh + } + } + } + return emptySecurityHolder +} + +// Close closes ObsClient. +func (obsClient ObsClient) Close() { + obsClient.httpClient = nil + obsClient.conf.transport.CloseIdleConnections() + obsClient.conf = nil +} + +// ListBuckets lists buckets. +// +// You can use this API to obtain the bucket list. In the list, bucket names are displayed in lexicographical order. +func (obsClient ObsClient) ListBuckets(input *ListBucketsInput, extensions ...extensionOptions) (output *ListBucketsOutput, err error) { + if input == nil { + input = &ListBucketsInput{} + } + output = &ListBucketsOutput{} + err = obsClient.doActionWithoutBucket("ListBuckets", HTTP_GET, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// CreateBucket creates a bucket. +// +// You can use this API to create a bucket and name it as you specify. The created bucket name must be unique in OBS. +func (obsClient ObsClient) CreateBucket(input *CreateBucketInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("CreateBucketInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("CreateBucket", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucket deletes a bucket. +// +// You can use this API to delete a bucket. The bucket to be deleted must be empty +// (containing no objects, noncurrent object versions, or part fragments). +func (obsClient ObsClient) DeleteBucket(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucket", HTTP_DELETE, bucketName, defaultSerializable, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketStoragePolicy sets bucket storage class. +// +// You can use this API to set storage class for bucket. +func (obsClient ObsClient) SetBucketStoragePolicy(input *SetBucketStoragePolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketStoragePolicyInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketStoragePolicy", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} +func (obsClient ObsClient) getBucketStoragePolicyS3(bucketName string, extensions []extensionOptions) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputS3 *getBucketStoragePolicyOutputS3 + outputS3 = &getBucketStoragePolicyOutputS3{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), outputS3, extensions) + if err != nil { + output = nil + return + } + output.BaseModel = outputS3.BaseModel + output.StorageClass = fmt.Sprintf("%s", outputS3.StorageClass) + return +} + +func (obsClient ObsClient) getBucketStoragePolicyObs(bucketName string, extensions []extensionOptions) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputObs *getBucketStoragePolicyOutputObs + outputObs = &getBucketStoragePolicyOutputObs{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageClass), outputObs, extensions) + if err != nil { + output = nil + return + } + output.BaseModel = outputObs.BaseModel + output.StorageClass = outputObs.StorageClass + return +} + +// GetBucketStoragePolicy gets bucket storage class. +// +// You can use this API to obtain the storage class of a bucket. +func (obsClient ObsClient) GetBucketStoragePolicy(bucketName string, extensions ...extensionOptions) (output *GetBucketStoragePolicyOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketStoragePolicyObs(bucketName, extensions) + } + return obsClient.getBucketStoragePolicyS3(bucketName, extensions) +} + +// ListObjects lists objects in a bucket. +// +// You can use this API to list objects in a bucket. By default, a maximum of 1000 objects are listed. +func (obsClient ObsClient) ListObjects(input *ListObjectsInput, extensions ...extensionOptions) (output *ListObjectsOutput, err error) { + if input == nil { + return nil, errors.New("ListObjectsInput is nil") + } + output = &ListObjectsOutput{} + err = obsClient.doActionWithBucket("ListObjects", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +// ListVersions lists versioning objects in a bucket. +// +// You can use this API to list versioning objects in a bucket. By default, a maximum of 1000 versioning objects are listed. +func (obsClient ObsClient) ListVersions(input *ListVersionsInput, extensions ...extensionOptions) (output *ListVersionsOutput, err error) { + if input == nil { + return nil, errors.New("ListVersionsInput is nil") + } + output = &ListVersionsOutput{} + err = obsClient.doActionWithBucket("ListVersions", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +// ListMultipartUploads lists the multipart uploads. +// +// You can use this API to list the multipart uploads that are initialized but not combined or aborted in a specified bucket. +func (obsClient ObsClient) ListMultipartUploads(input *ListMultipartUploadsInput, extensions ...extensionOptions) (output *ListMultipartUploadsOutput, err error) { + if input == nil { + return nil, errors.New("ListMultipartUploadsInput is nil") + } + output = &ListMultipartUploadsOutput{} + err = obsClient.doActionWithBucket("ListMultipartUploads", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketQuota sets the bucket quota. +// +// You can use this API to set the bucket quota. A bucket quota must be expressed in bytes and the maximum value is 2^63-1. +func (obsClient ObsClient) SetBucketQuota(input *SetBucketQuotaInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketQuotaInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketQuota", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketQuota gets the bucket quota. +// +// You can use this API to obtain the bucket quota. Value 0 indicates that no upper limit is set for the bucket quota. +func (obsClient ObsClient) GetBucketQuota(bucketName string, extensions ...extensionOptions) (output *GetBucketQuotaOutput, err error) { + output = &GetBucketQuotaOutput{} + err = obsClient.doActionWithBucket("GetBucketQuota", HTTP_GET, bucketName, newSubResourceSerial(SubResourceQuota), output, extensions) + if err != nil { + output = nil + } + return +} + +// HeadBucket checks whether a bucket exists. +// +// You can use this API to check whether a bucket exists. +func (obsClient ObsClient) HeadBucket(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("HeadBucket", HTTP_HEAD, bucketName, defaultSerializable, output, extensions) + if err != nil { + output = nil + } + return +} + +// HeadObject checks whether an object exists. +// +// You can use this API to check whether an object exists. +func (obsClient ObsClient) HeadObject(input *HeadObjectInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("HeadObjectInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("HeadObject", HTTP_HEAD, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketMetadata gets the metadata of a bucket. +// +// You can use this API to send a HEAD request to a bucket to obtain the bucket +// metadata such as the storage class and CORS rules (if set). +func (obsClient ObsClient) GetBucketMetadata(input *GetBucketMetadataInput, extensions ...extensionOptions) (output *GetBucketMetadataOutput, err error) { + output = &GetBucketMetadataOutput{} + err = obsClient.doActionWithBucket("GetBucketMetadata", HTTP_HEAD, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + ParseGetBucketMetadataOutput(output) + } + return +} + +// SetObjectMetadata sets object metadata. +func (obsClient ObsClient) SetObjectMetadata(input *SetObjectMetadataInput, extensions ...extensionOptions) (output *SetObjectMetadataOutput, err error) { + output = &SetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("SetObjectMetadata", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseSetObjectMetadataOutput(output) + } + return +} + +// GetBucketStorageInfo gets storage information about a bucket. +// +// You can use this API to obtain storage information about a bucket, including the +// bucket size and number of objects in the bucket. +func (obsClient ObsClient) GetBucketStorageInfo(bucketName string, extensions ...extensionOptions) (output *GetBucketStorageInfoOutput, err error) { + output = &GetBucketStorageInfoOutput{} + err = obsClient.doActionWithBucket("GetBucketStorageInfo", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageInfo), output, extensions) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) getBucketLocationS3(bucketName string, extensions []extensionOptions) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputS3 *getBucketLocationOutputS3 + outputS3 = &getBucketLocationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputS3, extensions) + if err != nil { + output = nil + } else { + output.BaseModel = outputS3.BaseModel + output.Location = outputS3.Location + } + return +} +func (obsClient ObsClient) getBucketLocationObs(bucketName string, extensions []extensionOptions) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputObs *getBucketLocationOutputObs + outputObs = &getBucketLocationOutputObs{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputObs, extensions) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Location = outputObs.Location + } + return +} + +// GetBucketLocation gets the location of a bucket. +// +// You can use this API to obtain the bucket location. +func (obsClient ObsClient) GetBucketLocation(bucketName string, extensions ...extensionOptions) (output *GetBucketLocationOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketLocationObs(bucketName, extensions) + } + return obsClient.getBucketLocationS3(bucketName, extensions) +} + +// SetBucketAcl sets the bucket ACL. +// +// You can use this API to set the ACL for a bucket. +func (obsClient ObsClient) SetBucketAcl(input *SetBucketAclInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketAclInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketAcl", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} +func (obsClient ObsClient) getBucketACLObs(bucketName string, extensions []extensionOptions) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + var outputObs *getBucketACLOutputObs + outputObs = &getBucketACLOutputObs{} + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), outputObs, extensions) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Owner = outputObs.Owner + output.Grants = make([]Grant, 0, len(outputObs.Grants)) + for _, valGrant := range outputObs.Grants { + tempOutput := Grant{} + tempOutput.Delivered = valGrant.Delivered + tempOutput.Permission = valGrant.Permission + tempOutput.Grantee.DisplayName = valGrant.Grantee.DisplayName + tempOutput.Grantee.ID = valGrant.Grantee.ID + tempOutput.Grantee.Type = valGrant.Grantee.Type + tempOutput.Grantee.URI = GroupAllUsers + + output.Grants = append(output.Grants, tempOutput) + } + } + return +} + +// GetBucketAcl gets the bucket ACL. +// +// You can use this API to obtain a bucket ACL. +func (obsClient ObsClient) GetBucketAcl(bucketName string, extensions ...extensionOptions) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketACLObs(bucketName, extensions) + } + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketPolicy sets the bucket policy. +// +// You can use this API to set a bucket policy. If the bucket already has a policy, the +// policy will be overwritten by the one specified in this request. +func (obsClient ObsClient) SetBucketPolicy(input *SetBucketPolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketPolicy is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketPolicy", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketPolicy gets the bucket policy. +// +// You can use this API to obtain the policy of a bucket. +func (obsClient ObsClient) GetBucketPolicy(bucketName string, extensions ...extensionOptions) (output *GetBucketPolicyOutput, err error) { + output = &GetBucketPolicyOutput{} + err = obsClient.doActionWithBucketV2("GetBucketPolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourcePolicy), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketPolicy deletes the bucket policy. +// +// You can use this API to delete the policy of a bucket. +func (obsClient ObsClient) DeleteBucketPolicy(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketPolicy", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourcePolicy), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketCors sets CORS rules for a bucket. +// +// You can use this API to set CORS rules for a bucket to allow client browsers to send cross-origin requests. +func (obsClient ObsClient) SetBucketCors(input *SetBucketCorsInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketCorsInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketCors", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketCors gets CORS rules of a bucket. +// +// You can use this API to obtain the CORS rules of a specified bucket. +func (obsClient ObsClient) GetBucketCors(bucketName string, extensions ...extensionOptions) (output *GetBucketCorsOutput, err error) { + output = &GetBucketCorsOutput{} + err = obsClient.doActionWithBucket("GetBucketCors", HTTP_GET, bucketName, newSubResourceSerial(SubResourceCors), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketCors deletes CORS rules of a bucket. +// +// You can use this API to delete the CORS rules of a specified bucket. +func (obsClient ObsClient) DeleteBucketCors(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketCors", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceCors), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketVersioning sets the versioning status for a bucket. +// +// You can use this API to set the versioning status for a bucket. +func (obsClient ObsClient) SetBucketVersioning(input *SetBucketVersioningInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketVersioningInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketVersioning", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketVersioning gets the versioning status of a bucket. +// +// You can use this API to obtain the versioning status of a bucket. +func (obsClient ObsClient) GetBucketVersioning(bucketName string, extensions ...extensionOptions) (output *GetBucketVersioningOutput, err error) { + output = &GetBucketVersioningOutput{} + err = obsClient.doActionWithBucket("GetBucketVersioning", HTTP_GET, bucketName, newSubResourceSerial(SubResourceVersioning), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketWebsiteConfiguration sets website hosting for a bucket. +// +// You can use this API to set website hosting for a bucket. +func (obsClient ObsClient) SetBucketWebsiteConfiguration(input *SetBucketWebsiteConfigurationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketWebsiteConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketWebsiteConfiguration", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketWebsiteConfiguration gets the website hosting settings of a bucket. +// +// You can use this API to obtain the website hosting settings of a bucket. +func (obsClient ObsClient) GetBucketWebsiteConfiguration(bucketName string, extensions ...extensionOptions) (output *GetBucketWebsiteConfigurationOutput, err error) { + output = &GetBucketWebsiteConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketWebsiteConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceWebsite), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketWebsiteConfiguration deletes the website hosting settings of a bucket. +// +// You can use this API to delete the website hosting settings of a bucket. +func (obsClient ObsClient) DeleteBucketWebsiteConfiguration(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketWebsiteConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceWebsite), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketLoggingConfiguration sets the bucket logging. +// +// You can use this API to configure access logging for a bucket. +func (obsClient ObsClient) SetBucketLoggingConfiguration(input *SetBucketLoggingConfigurationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketLoggingConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketLoggingConfiguration", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketLoggingConfiguration gets the logging settings of a bucket. +// +// You can use this API to obtain the access logging settings of a bucket. +func (obsClient ObsClient) GetBucketLoggingConfiguration(bucketName string, extensions ...extensionOptions) (output *GetBucketLoggingConfigurationOutput, err error) { + output = &GetBucketLoggingConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketLoggingConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLogging), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketLifecycleConfiguration sets lifecycle rules for a bucket. +// +// You can use this API to set lifecycle rules for a bucket, to periodically transit +// storage classes of objects and delete objects in the bucket. +func (obsClient ObsClient) SetBucketLifecycleConfiguration(input *SetBucketLifecycleConfigurationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketLifecycleConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketLifecycleConfiguration", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketLifecycleConfiguration gets lifecycle rules of a bucket. +// +// You can use this API to obtain the lifecycle rules of a bucket. +func (obsClient ObsClient) GetBucketLifecycleConfiguration(bucketName string, extensions ...extensionOptions) (output *GetBucketLifecycleConfigurationOutput, err error) { + output = &GetBucketLifecycleConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketLifecycleConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLifecycle), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketLifecycleConfiguration deletes lifecycle rules of a bucket. +// +// You can use this API to delete all lifecycle rules of a bucket. +func (obsClient ObsClient) DeleteBucketLifecycleConfiguration(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketLifecycleConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceLifecycle), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketTagging sets bucket tags. +// +// You can use this API to set bucket tags. +func (obsClient ObsClient) SetBucketTagging(input *SetBucketTaggingInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketTaggingInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketTagging", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketTagging gets bucket tags. +// +// You can use this API to obtain the tags of a specified bucket. +func (obsClient ObsClient) GetBucketTagging(bucketName string, extensions ...extensionOptions) (output *GetBucketTaggingOutput, err error) { + output = &GetBucketTaggingOutput{} + err = obsClient.doActionWithBucket("GetBucketTagging", HTTP_GET, bucketName, newSubResourceSerial(SubResourceTagging), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketTagging deletes bucket tags. +// +// You can use this API to delete the tags of a specified bucket. +func (obsClient ObsClient) DeleteBucketTagging(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketTagging", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceTagging), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketNotification sets event notification for a bucket. +// +// You can use this API to configure event notification for a bucket. You will be notified of all +// specified operations performed on the bucket. +func (obsClient ObsClient) SetBucketNotification(input *SetBucketNotificationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketNotificationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketNotification", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketNotification gets event notification settings of a bucket. +// +// You can use this API to obtain the event notification configuration of a bucket. +func (obsClient ObsClient) GetBucketNotification(bucketName string, extensions ...extensionOptions) (output *GetBucketNotificationOutput, err error) { + if obsClient.conf.signature != SignatureObs { + return obsClient.getBucketNotificationS3(bucketName, extensions) + } + output = &GetBucketNotificationOutput{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), output, extensions) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) getBucketNotificationS3(bucketName string, extensions []extensionOptions) (output *GetBucketNotificationOutput, err error) { + outputS3 := &getBucketNotificationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), outputS3, extensions) + if err != nil { + return nil, err + } + + output = &GetBucketNotificationOutput{} + output.BaseModel = outputS3.BaseModel + topicConfigurations := make([]TopicConfiguration, 0, len(outputS3.TopicConfigurations)) + for _, topicConfigurationS3 := range outputS3.TopicConfigurations { + topicConfiguration := TopicConfiguration{} + topicConfiguration.ID = topicConfigurationS3.ID + topicConfiguration.Topic = topicConfigurationS3.Topic + topicConfiguration.FilterRules = topicConfigurationS3.FilterRules + + events := make([]EventType, 0, len(topicConfigurationS3.Events)) + for _, event := range topicConfigurationS3.Events { + events = append(events, ParseStringToEventType(event)) + } + topicConfiguration.Events = events + topicConfigurations = append(topicConfigurations, topicConfiguration) + } + output.TopicConfigurations = topicConfigurations + return +} + +// DeleteObject deletes an object. +// +// You can use this API to delete an object from a specified bucket. +func (obsClient ObsClient) DeleteObject(input *DeleteObjectInput, extensions ...extensionOptions) (output *DeleteObjectOutput, err error) { + if input == nil { + return nil, errors.New("DeleteObjectInput is nil") + } + output = &DeleteObjectOutput{} + err = obsClient.doActionWithBucketAndKey("DeleteObject", HTTP_DELETE, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseDeleteObjectOutput(output) + } + return +} + +// DeleteObjects deletes objects in a batch. +// +// You can use this API to batch delete objects from a specified bucket. +func (obsClient ObsClient) DeleteObjects(input *DeleteObjectsInput, extensions ...extensionOptions) (output *DeleteObjectsOutput, err error) { + if input == nil { + return nil, errors.New("DeleteObjectsInput is nil") + } + output = &DeleteObjectsOutput{} + err = obsClient.doActionWithBucket("DeleteObjects", HTTP_POST, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetObjectAcl sets ACL for an object. +// +// You can use this API to set the ACL for an object in a specified bucket. +func (obsClient ObsClient) SetObjectAcl(input *SetObjectAclInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetObjectAclInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("SetObjectAcl", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetObjectAcl gets the ACL of an object. +// +// You can use this API to obtain the ACL of an object in a specified bucket. +func (obsClient ObsClient) GetObjectAcl(input *GetObjectAclInput, extensions ...extensionOptions) (output *GetObjectAclOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectAclInput is nil") + } + output = &GetObjectAclOutput{} + err = obsClient.doActionWithBucketAndKey("GetObjectAcl", HTTP_GET, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + if versionID, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionID[0] + } + } + return +} + +// RestoreObject restores an object. +func (obsClient ObsClient) RestoreObject(input *RestoreObjectInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("RestoreObjectInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("RestoreObject", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetObjectMetadata gets object metadata. +// +// You can use this API to send a HEAD request to the object of a specified bucket to obtain its metadata. +func (obsClient ObsClient) GetObjectMetadata(input *GetObjectMetadataInput, extensions ...extensionOptions) (output *GetObjectMetadataOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectMetadataInput is nil") + } + output = &GetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("GetObjectMetadata", HTTP_HEAD, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseGetObjectMetadataOutput(output) + } + return +} + +// GetObject downloads object. +// +// You can use this API to download an object in a specified bucket. +func (obsClient ObsClient) GetObject(input *GetObjectInput, extensions ...extensionOptions) (output *GetObjectOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectInput is nil") + } + output = &GetObjectOutput{} + err = obsClient.doActionWithBucketAndKey("GetObject", HTTP_GET, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseGetObjectOutput(output) + } + return +} + +// PutObject uploads an object to the specified bucket. +func (obsClient ObsClient) PutObject(input *PutObjectInput, extensions ...extensionOptions) (output *PutObjectOutput, err error) { + if input == nil { + return nil, errors.New("PutObjectInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + input.ContentType = contentType + } + } + output = &PutObjectOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if input.ContentLength > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.ContentLength} + } + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +func (obsClient ObsClient) getContentType(input *PutObjectInput, sourceFile string) (contentType string) { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + return contentType + } + if contentType, ok := mimeTypes[strings.ToLower(sourceFile[strings.LastIndex(sourceFile, ".")+1:])]; ok { + return contentType + } + return +} + +func (obsClient ObsClient) isGetContentType(input *PutObjectInput) bool { + if input.ContentType == "" && input.Key != "" { + return true + } + return false +} + +// PutFile uploads a file to the specified bucket. +func (obsClient ObsClient) PutFile(input *PutFileInput, extensions ...extensionOptions) (output *PutObjectOutput, err error) { + if input == nil { + return nil, errors.New("PutFileInput is nil") + } + + var body io.Reader + sourceFile := strings.TrimSpace(input.SourceFile) + if sourceFile != "" { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() + + stat, _err := fd.Stat() + if _err != nil { + err = _err + return nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + if input.ContentLength > 0 { + if input.ContentLength > stat.Size() { + input.ContentLength = stat.Size() + } + fileReaderWrapper.totalCount = input.ContentLength + } else { + fileReaderWrapper.totalCount = stat.Size() + } + body = fileReaderWrapper + } + + _input := &PutObjectInput{} + _input.PutObjectBasicInput = input.PutObjectBasicInput + _input.Body = body + + if obsClient.isGetContentType(_input) { + _input.ContentType = obsClient.getContentType(_input, sourceFile) + } + + output = &PutObjectOutput{} + err = obsClient.doActionWithBucketAndKey("PutFile", HTTP_PUT, _input.Bucket, _input.Key, _input, output, extensions) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +// CopyObject creates a copy for an existing object. +// +// You can use this API to create a copy for an object in a specified bucket. +func (obsClient ObsClient) CopyObject(input *CopyObjectInput, extensions ...extensionOptions) (output *CopyObjectOutput, err error) { + if input == nil { + return nil, errors.New("CopyObjectInput is nil") + } + + if strings.TrimSpace(input.CopySourceBucket) == "" { + return nil, errors.New("Source bucket is empty") + } + if strings.TrimSpace(input.CopySourceKey) == "" { + return nil, errors.New("Source key is empty") + } + + output = &CopyObjectOutput{} + err = obsClient.doActionWithBucketAndKey("CopyObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseCopyObjectOutput(output) + } + return +} + +// AbortMultipartUpload aborts a multipart upload in a specified bucket by using the multipart upload ID. +func (obsClient ObsClient) AbortMultipartUpload(input *AbortMultipartUploadInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("AbortMultipartUploadInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("AbortMultipartUpload", HTTP_DELETE, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// InitiateMultipartUpload initializes a multipart upload. +func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploadInput, extensions ...extensionOptions) (output *InitiateMultipartUploadOutput, err error) { + if input == nil { + return nil, errors.New("InitiateMultipartUploadInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + input.ContentType = contentType + } + } + + output = &InitiateMultipartUploadOutput{} + err = obsClient.doActionWithBucketAndKey("InitiateMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseInitiateMultipartUploadOutput(output) + } + return +} + +// UploadPart uploads a part to a specified bucket by using a specified multipart upload ID. +// +// After a multipart upload is initialized, you can use this API to upload a part to a specified bucket +// by using the multipart upload ID. Except for the last uploaded part whose size ranges from 0 to 5 GB, +// sizes of the other parts range from 100 KB to 5 GB. The upload part ID ranges from 1 to 10000. +func (obsClient ObsClient) UploadPart(_input *UploadPartInput, extensions ...extensionOptions) (output *UploadPartOutput, err error) { + if _input == nil { + return nil, errors.New("UploadPartInput is nil") + } + + if _input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + + input := &UploadPartInput{} + input.Bucket = _input.Bucket + input.Key = _input.Key + input.PartNumber = _input.PartNumber + input.UploadId = _input.UploadId + input.ContentMD5 = _input.ContentMD5 + input.SourceFile = _input.SourceFile + input.Offset = _input.Offset + input.PartSize = _input.PartSize + input.SseHeader = _input.SseHeader + input.Body = _input.Body + + output = &UploadPartOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if _, ok := input.Body.(*readerWrapper); !ok && input.PartSize > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.PartSize} + } + } else if sourceFile := strings.TrimSpace(input.SourceFile); sourceFile != "" { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() + + stat, _err := fd.Stat() + if _err != nil { + err = _err + return nil, err + } + fileSize := stat.Size() + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + + if input.Offset < 0 || input.Offset > fileSize { + input.Offset = 0 + } + + if input.PartSize <= 0 || input.PartSize > (fileSize-input.Offset) { + input.PartSize = fileSize - input.Offset + } + fileReaderWrapper.totalCount = input.PartSize + if _, err = fd.Seek(input.Offset, io.SeekStart); err != nil { + return nil, err + } + input.Body = fileReaderWrapper + repeatable = true + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } + if err != nil { + output = nil + } else { + ParseUploadPartOutput(output) + output.PartNumber = input.PartNumber + } + return +} + +// CompleteMultipartUpload combines the uploaded parts in a specified bucket by using the multipart upload ID. +func (obsClient ObsClient) CompleteMultipartUpload(input *CompleteMultipartUploadInput, extensions ...extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + if input == nil { + return nil, errors.New("CompleteMultipartUploadInput is nil") + } + + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + + var parts partSlice = input.Parts + sort.Sort(parts) + + output = &CompleteMultipartUploadOutput{} + err = obsClient.doActionWithBucketAndKey("CompleteMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseCompleteMultipartUploadOutput(output) + } + return +} + +// ListParts lists the uploaded parts in a bucket by using the multipart upload ID. +func (obsClient ObsClient) ListParts(input *ListPartsInput, extensions ...extensionOptions) (output *ListPartsOutput, err error) { + if input == nil { + return nil, errors.New("ListPartsInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + output = &ListPartsOutput{} + err = obsClient.doActionWithBucketAndKey("ListParts", HTTP_GET, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// CopyPart copy a part to a specified bucket by using a specified multipart upload ID. +// +// After a multipart upload is initialized, you can use this API to copy a part to a specified bucket by using the multipart upload ID. +func (obsClient ObsClient) CopyPart(input *CopyPartInput, extensions ...extensionOptions) (output *CopyPartOutput, err error) { + if input == nil { + return nil, errors.New("CopyPartInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + if strings.TrimSpace(input.CopySourceBucket) == "" { + return nil, errors.New("Source bucket is empty") + } + if strings.TrimSpace(input.CopySourceKey) == "" { + return nil, errors.New("Source key is empty") + } + + output = &CopyPartOutput{} + err = obsClient.doActionWithBucketAndKey("CopyPart", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseCopyPartOutput(output) + output.PartNumber = input.PartNumber + } + return +} + +// SetBucketRequestPayment sets requester-pays setting for a bucket. +func (obsClient ObsClient) SetBucketRequestPayment(input *SetBucketRequestPaymentInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketRequestPaymentInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketRequestPayment", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketRequestPayment gets requester-pays setting of a bucket. +func (obsClient ObsClient) GetBucketRequestPayment(bucketName string, extensions ...extensionOptions) (output *GetBucketRequestPaymentOutput, err error) { + output = &GetBucketRequestPaymentOutput{} + err = obsClient.doActionWithBucket("GetBucketRequestPayment", HTTP_GET, bucketName, newSubResourceSerial(SubResourceRequestPayment), output, extensions) + if err != nil { + output = nil + } + return +} + +// UploadFile resume uploads. +// +// This API is an encapsulated and enhanced version of multipart upload, and aims to eliminate large file +// upload failures caused by poor network conditions and program breakdowns. +func (obsClient ObsClient) UploadFile(input *UploadFileInput, extensions ...extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + if input.EnableCheckpoint && input.CheckpointFile == "" { + input.CheckpointFile = input.UploadFile + ".uploadfile_record" + } + + if input.TaskNum <= 0 { + input.TaskNum = 1 + } + if input.PartSize < MIN_PART_SIZE { + input.PartSize = MIN_PART_SIZE + } else if input.PartSize > MAX_PART_SIZE { + input.PartSize = MAX_PART_SIZE + } + + output, err = obsClient.resumeUpload(input, extensions) + return +} + +// DownloadFile resume downloads. +// +// This API is an encapsulated and enhanced version of partial download, and aims to eliminate large file +// download failures caused by poor network conditions and program breakdowns. +func (obsClient ObsClient) DownloadFile(input *DownloadFileInput, extensions ...extensionOptions) (output *GetObjectMetadataOutput, err error) { + if input.DownloadFile == "" { + input.DownloadFile = input.Key + } + + if input.EnableCheckpoint && input.CheckpointFile == "" { + input.CheckpointFile = input.DownloadFile + ".downloadfile_record" + } + + if input.TaskNum <= 0 { + input.TaskNum = 1 + } + if input.PartSize <= 0 { + input.PartSize = DEFAULT_PART_SIZE + } + + output, err = obsClient.resumeDownload(input, extensions) + return +} + +// SetBucketFetchPolicy sets the bucket fetch policy. +// +// You can use this API to set a bucket fetch policy. +func (obsClient ObsClient) SetBucketFetchPolicy(input *SetBucketFetchPolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketFetchPolicyInput is nil") + } + if strings.TrimSpace(string(input.Status)) == "" { + return nil, errors.New("Fetch policy status is empty") + } + if strings.TrimSpace(input.Agency) == "" { + return nil, errors.New("Fetch policy agency is empty") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("SetBucketFetchPolicy", HTTP_PUT, input.Bucket, string(objectKeyExtensionPolicy), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketFetchPolicy gets the bucket fetch policy. +// +// You can use this API to obtain the fetch policy of a bucket. +func (obsClient ObsClient) GetBucketFetchPolicy(input *GetBucketFetchPolicyInput, extensions ...extensionOptions) (output *GetBucketFetchPolicyOutput, err error) { + if input == nil { + return nil, errors.New("GetBucketFetchPolicyInput is nil") + } + output = &GetBucketFetchPolicyOutput{} + err = obsClient.doActionWithBucketAndKeyV2("GetBucketFetchPolicy", HTTP_GET, input.Bucket, string(objectKeyExtensionPolicy), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketFetchPolicy deletes the bucket fetch policy. +// +// You can use this API to delete the fetch policy of a bucket. +func (obsClient ObsClient) DeleteBucketFetchPolicy(input *DeleteBucketFetchPolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("DeleteBucketFetchPolicyInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("DeleteBucketFetchPolicy", HTTP_DELETE, input.Bucket, string(objectKeyExtensionPolicy), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketFetchJob sets the bucket fetch job. +// +// You can use this API to set a bucket fetch job. +func (obsClient ObsClient) SetBucketFetchJob(input *SetBucketFetchJobInput, extensions ...extensionOptions) (output *SetBucketFetchJobOutput, err error) { + if input == nil { + return nil, errors.New("SetBucketFetchJobInput is nil") + } + if strings.TrimSpace(input.URL) == "" { + return nil, errors.New("URL is empty") + } + output = &SetBucketFetchJobOutput{} + err = obsClient.doActionWithBucketAndKeyV2("SetBucketFetchJob", HTTP_POST, input.Bucket, string(objectKeyAsyncFetchJob), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketFetchJob gets the bucket fetch job. +// +// You can use this API to obtain the fetch job of a bucket. +func (obsClient ObsClient) GetBucketFetchJob(input *GetBucketFetchJobInput, extensions ...extensionOptions) (output *GetBucketFetchJobOutput, err error) { + if input == nil { + return nil, errors.New("GetBucketFetchJobInput is nil") + } + if strings.TrimSpace(input.JobID) == "" { + return nil, errors.New("JobID is empty") + } + output = &GetBucketFetchJobOutput{} + err = obsClient.doActionWithBucketAndKeyV2("GetBucketFetchJob", HTTP_GET, input.Bucket, string(objectKeyAsyncFetchJob)+"/"+input.JobID, input, output, extensions) + if err != nil { + output = nil + } + return +} diff --git a/cve-vulner-manager/obs/conf.go b/cve-vulner-manager/obs/conf.go new file mode 100644 index 0000000000000000000000000000000000000000..0c4eb89bbfdfb515a34868e84184f7089fe9f9af --- /dev/null +++ b/cve-vulner-manager/obs/conf.go @@ -0,0 +1,493 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +type urlHolder struct { + scheme string + host string + port int +} + +type config struct { + securityProviders []securityProvider + urlHolder *urlHolder + pathStyle bool + cname bool + sslVerify bool + endpoint string + signature SignatureType + region string + connectTimeout int + socketTimeout int + headerTimeout int + idleConnTimeout int + finalTimeout int + maxRetryCount int + proxyURL string + maxConnsPerHost int + pemCerts []byte + transport *http.Transport + ctx context.Context + maxRedirectCount int + userAgent string + enableCompression bool +} + +func (conf config) String() string { + return fmt.Sprintf("[endpoint:%s, signature:%s, pathStyle:%v, region:%s"+ + "\nconnectTimeout:%d, socketTimeout:%dheaderTimeout:%d, idleConnTimeout:%d"+ + "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, maxRedirectCount:%d]", + conf.endpoint, conf.signature, conf.pathStyle, conf.region, + conf.connectTimeout, conf.socketTimeout, conf.headerTimeout, conf.idleConnTimeout, + conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.maxRedirectCount, + ) +} + +type configurer func(conf *config) + +func WithSecurityProviders(sps ...securityProvider) configurer { + return func(conf *config) { + for _, sp := range sps { + if sp != nil { + conf.securityProviders = append(conf.securityProviders, sp) + } + } + } +} + +// WithSslVerify is a wrapper for WithSslVerifyAndPemCerts. +func WithSslVerify(sslVerify bool) configurer { + return WithSslVerifyAndPemCerts(sslVerify, nil) +} + +// WithSslVerifyAndPemCerts is a configurer for ObsClient to set conf.sslVerify and conf.pemCerts. +func WithSslVerifyAndPemCerts(sslVerify bool, pemCerts []byte) configurer { + return func(conf *config) { + conf.sslVerify = sslVerify + conf.pemCerts = pemCerts + } +} + +// WithHeaderTimeout is a configurer for ObsClient to set the timeout period of obtaining the response headers. +func WithHeaderTimeout(headerTimeout int) configurer { + return func(conf *config) { + conf.headerTimeout = headerTimeout + } +} + +// WithProxyUrl is a configurer for ObsClient to set HTTP proxy. +func WithProxyUrl(proxyURL string) configurer { + return func(conf *config) { + conf.proxyURL = proxyURL + } +} + +// WithMaxConnections is a configurer for ObsClient to set the maximum number of idle HTTP connections. +func WithMaxConnections(maxConnsPerHost int) configurer { + return func(conf *config) { + conf.maxConnsPerHost = maxConnsPerHost + } +} + +// WithPathStyle is a configurer for ObsClient. +func WithPathStyle(pathStyle bool) configurer { + return func(conf *config) { + conf.pathStyle = pathStyle + } +} + +// WithSignature is a configurer for ObsClient. +func WithSignature(signature SignatureType) configurer { + return func(conf *config) { + conf.signature = signature + } +} + +// WithRegion is a configurer for ObsClient. +func WithRegion(region string) configurer { + return func(conf *config) { + conf.region = region + } +} + +// WithConnectTimeout is a configurer for ObsClient to set timeout period for establishing +// an http/https connection, in seconds. +func WithConnectTimeout(connectTimeout int) configurer { + return func(conf *config) { + conf.connectTimeout = connectTimeout + } +} + +// WithSocketTimeout is a configurer for ObsClient to set the timeout duration for transmitting data at +// the socket layer, in seconds. +func WithSocketTimeout(socketTimeout int) configurer { + return func(conf *config) { + conf.socketTimeout = socketTimeout + } +} + +// WithIdleConnTimeout is a configurer for ObsClient to set the timeout period of an idle HTTP connection +// in the connection pool, in seconds. +func WithIdleConnTimeout(idleConnTimeout int) configurer { + return func(conf *config) { + conf.idleConnTimeout = idleConnTimeout + } +} + +// WithMaxRetryCount is a configurer for ObsClient to set the maximum number of retries when an HTTP/HTTPS connection is abnormal. +func WithMaxRetryCount(maxRetryCount int) configurer { + return func(conf *config) { + conf.maxRetryCount = maxRetryCount + } +} + +// WithSecurityToken is a configurer for ObsClient to set the security token in the temporary access keys. +func WithSecurityToken(securityToken string) configurer { + return func(conf *config) { + for _, sp := range conf.securityProviders { + if bsp, ok := sp.(*BasicSecurityProvider); ok { + sh := bsp.getSecurity() + bsp.refresh(sh.ak, sh.sk, securityToken) + break + } + } + } +} + +// WithHttpTransport is a configurer for ObsClient to set the customized http Transport. +func WithHttpTransport(transport *http.Transport) configurer { + return func(conf *config) { + conf.transport = transport + } +} + +// WithRequestContext is a configurer for ObsClient to set the context for each HTTP request. +func WithRequestContext(ctx context.Context) configurer { + return func(conf *config) { + conf.ctx = ctx + } +} + +// WithCustomDomainName is a configurer for ObsClient. +func WithCustomDomainName(cname bool) configurer { + return func(conf *config) { + conf.cname = cname + } +} + +// WithMaxRedirectCount is a configurer for ObsClient to set the maximum number of times that the request is redirected. +func WithMaxRedirectCount(maxRedirectCount int) configurer { + return func(conf *config) { + conf.maxRedirectCount = maxRedirectCount + } +} + +// WithUserAgent is a configurer for ObsClient to set the User-Agent. +func WithUserAgent(userAgent string) configurer { + return func(conf *config) { + conf.userAgent = userAgent + } +} + +// WithEnableCompression is a configurer for ObsClient to set the Transport.DisableCompression. +func WithEnableCompression(enableCompression bool) configurer { + return func(conf *config) { + conf.enableCompression = enableCompression + } +} + +func (conf *config) prepareConfig() { + if conf.connectTimeout <= 0 { + conf.connectTimeout = DEFAULT_CONNECT_TIMEOUT + } + + if conf.socketTimeout <= 0 { + conf.socketTimeout = DEFAULT_SOCKET_TIMEOUT + } + + conf.finalTimeout = conf.socketTimeout * 10 + + if conf.headerTimeout <= 0 { + conf.headerTimeout = DEFAULT_HEADER_TIMEOUT + } + + if conf.idleConnTimeout < 0 { + conf.idleConnTimeout = DEFAULT_IDLE_CONN_TIMEOUT + } + + if conf.maxRetryCount < 0 { + conf.maxRetryCount = DEFAULT_MAX_RETRY_COUNT + } + + if conf.maxConnsPerHost <= 0 { + conf.maxConnsPerHost = DEFAULT_MAX_CONN_PER_HOST + } + + if conf.maxRedirectCount < 0 { + conf.maxRedirectCount = DEFAULT_MAX_REDIRECT_COUNT + } +} + +func (conf *config) initConfigWithDefault() error { + conf.endpoint = strings.TrimSpace(conf.endpoint) + if conf.endpoint == "" { + return errors.New("endpoint is not set") + } + + if index := strings.Index(conf.endpoint, "?"); index > 0 { + conf.endpoint = conf.endpoint[:index] + } + + for strings.LastIndex(conf.endpoint, "/") == len(conf.endpoint)-1 { + conf.endpoint = conf.endpoint[:len(conf.endpoint)-1] + } + + if conf.signature == "" { + conf.signature = DEFAULT_SIGNATURE + } + + urlHolder := &urlHolder{} + var address string + if strings.HasPrefix(conf.endpoint, "https://") { + urlHolder.scheme = "https" + address = conf.endpoint[len("https://"):] + } else if strings.HasPrefix(conf.endpoint, "http://") { + urlHolder.scheme = "http" + address = conf.endpoint[len("http://"):] + } else { + urlHolder.scheme = "https" + address = conf.endpoint + } + + addr := strings.Split(address, ":") + if len(addr) == 2 { + if port, err := strconv.Atoi(addr[1]); err == nil { + urlHolder.port = port + } + } + urlHolder.host = addr[0] + if urlHolder.port == 0 { + if urlHolder.scheme == "https" { + urlHolder.port = 443 + } else { + urlHolder.port = 80 + } + } + + if IsIP(urlHolder.host) { + conf.pathStyle = true + } + + conf.urlHolder = urlHolder + + conf.region = strings.TrimSpace(conf.region) + if conf.region == "" { + conf.region = DEFAULT_REGION + } + + conf.prepareConfig() + conf.proxyURL = strings.TrimSpace(conf.proxyURL) + return nil +} + +func (conf *config) getTransport() error { + if conf.transport == nil { + conf.transport = &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(network, addr, time.Second*time.Duration(conf.connectTimeout)) + if err != nil { + return nil, err + } + return getConnDelegate(conn, conf.socketTimeout, conf.finalTimeout), nil + }, + MaxIdleConns: conf.maxConnsPerHost, + MaxIdleConnsPerHost: conf.maxConnsPerHost, + ResponseHeaderTimeout: time.Second * time.Duration(conf.headerTimeout), + IdleConnTimeout: time.Second * time.Duration(conf.idleConnTimeout), + } + + if conf.proxyURL != "" { + proxyURL, err := url.Parse(conf.proxyURL) + if err != nil { + return err + } + conf.transport.Proxy = http.ProxyURL(proxyURL) + } + + tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify} + if conf.sslVerify && conf.pemCerts != nil { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(conf.pemCerts) + tlsConfig.RootCAs = pool + } + + conf.transport.TLSClientConfig = tlsConfig + conf.transport.DisableCompression = !conf.enableCompression + } + + return nil +} + +func checkRedirectFunc(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse +} + +// DummyQueryEscape return the input string. +func DummyQueryEscape(s string) string { + return s +} + +func (conf *config) prepareBaseURL(bucketName string) (requestURL string, canonicalizedURL string) { + urlHolder := conf.urlHolder + if conf.cname { + requestURL = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + if conf.signature == "v4" { + canonicalizedURL = "/" + } else { + canonicalizedURL = "/" + urlHolder.host + "/" + } + } else { + if bucketName == "" { + requestURL = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + canonicalizedURL = "/" + } else { + if conf.pathStyle { + requestURL = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) + canonicalizedURL = "/" + bucketName + } else { + requestURL = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) + if conf.signature == "v2" || conf.signature == "OBS" { + canonicalizedURL = "/" + bucketName + "/" + } else { + canonicalizedURL = "/" + } + } + } + } + return +} + +func (conf *config) prepareObjectKey(escape bool, objectKey string, escapeFunc func(s string) string) (encodeObjectKey string) { + if escape { + tempKey := []rune(objectKey) + result := make([]string, 0, len(tempKey)) + for _, value := range tempKey { + if string(value) == "/" { + result = append(result, string(value)) + } else { + if string(value) == " " { + result = append(result, url.PathEscape(string(value))) + } else { + result = append(result, url.QueryEscape(string(value))) + } + } + } + encodeObjectKey = strings.Join(result, "") + } else { + encodeObjectKey = escapeFunc(objectKey) + } + return +} + +func (conf *config) prepareEscapeFunc(escape bool) (escapeFunc func(s string) string) { + if escape { + return url.QueryEscape + } + return DummyQueryEscape +} + +func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string, escape bool) (requestURL string, canonicalizedURL string) { + + requestURL, canonicalizedURL = conf.prepareBaseURL(bucketName) + var escapeFunc func(s string) string + escapeFunc = conf.prepareEscapeFunc(escape) + + if objectKey != "" { + var encodeObjectKey string + encodeObjectKey = conf.prepareObjectKey(escape, objectKey, escapeFunc) + requestURL += "/" + encodeObjectKey + if !strings.HasSuffix(canonicalizedURL, "/") { + canonicalizedURL += "/" + } + canonicalizedURL += encodeObjectKey + } + + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, strings.TrimSpace(key)) + } + sort.Strings(keys) + i := 0 + + for index, key := range keys { + if index == 0 { + requestURL += "?" + } else { + requestURL += "&" + } + _key := url.QueryEscape(key) + requestURL += _key + + _value := params[key] + if conf.signature == "v4" { + requestURL += "=" + url.QueryEscape(_value) + } else { + if _value != "" { + requestURL += "=" + url.QueryEscape(_value) + _value = "=" + _value + } else { + _value = "" + } + lowerKey := strings.ToLower(key) + _, ok := allowedResourceParameterNames[lowerKey] + prefixHeader := HEADER_PREFIX + isObs := conf.signature == SignatureObs + if isObs { + prefixHeader = HEADER_PREFIX_OBS + } + ok = ok || strings.HasPrefix(lowerKey, prefixHeader) + if ok { + if i == 0 { + canonicalizedURL += "?" + } else { + canonicalizedURL += "&" + } + canonicalizedURL += getQueryURL(_key, _value) + i++ + } + } + } + return +} + +func getQueryURL(key, value string) string { + queryURL := "" + queryURL += key + queryURL += value + return queryURL +} diff --git a/cve-vulner-manager/obs/const.go b/cve-vulner-manager/obs/const.go new file mode 100644 index 0000000000000000000000000000000000000000..5e8391cab2d3cde0ce664d80d4cbde82a9dc650f --- /dev/null +++ b/cve-vulner-manager/obs/const.go @@ -0,0 +1,948 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +const ( + obsSdkVersion = "3.20.11" + USER_AGENT = "obs-sdk-go/" + obsSdkVersion + HEADER_PREFIX = "x-amz-" + HEADER_PREFIX_META = "x-amz-meta-" + HEADER_PREFIX_OBS = "x-obs-" + HEADER_PREFIX_META_OBS = "x-obs-meta-" + HEADER_DATE_AMZ = "x-amz-date" + HEADER_DATE_OBS = "x-obs-date" + HEADER_STS_TOKEN_AMZ = "x-amz-security-token" + HEADER_STS_TOKEN_OBS = "x-obs-security-token" + HEADER_ACCESSS_KEY_AMZ = "AWSAccessKeyId" + PREFIX_META = "meta-" + + HEADER_CONTENT_SHA256_AMZ = "x-amz-content-sha256" + HEADER_ACL_AMZ = "x-amz-acl" + HEADER_ACL_OBS = "x-obs-acl" + HEADER_ACL = "acl" + HEADER_LOCATION_AMZ = "location" + HEADER_BUCKET_LOCATION_OBS = "bucket-location" + HEADER_COPY_SOURCE = "copy-source" + HEADER_COPY_SOURCE_RANGE = "copy-source-range" + HEADER_RANGE = "Range" + HEADER_STORAGE_CLASS = "x-default-storage-class" + HEADER_STORAGE_CLASS_OBS = "x-obs-storage-class" + HEADER_VERSION_OBS = "version" + HEADER_GRANT_READ_OBS = "grant-read" + HEADER_GRANT_WRITE_OBS = "grant-write" + HEADER_GRANT_READ_ACP_OBS = "grant-read-acp" + HEADER_GRANT_WRITE_ACP_OBS = "grant-write-acp" + HEADER_GRANT_FULL_CONTROL_OBS = "grant-full-control" + HEADER_GRANT_READ_DELIVERED_OBS = "grant-read-delivered" + HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS = "grant-full-control-delivered" + HEADER_REQUEST_ID = "request-id" + HEADER_BUCKET_REGION = "bucket-region" + HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN = "access-control-allow-origin" + HEADER_ACCESS_CONRTOL_ALLOW_HEADERS = "access-control-allow-headers" + HEADER_ACCESS_CONRTOL_MAX_AGE = "access-control-max-age" + HEADER_ACCESS_CONRTOL_ALLOW_METHODS = "access-control-allow-methods" + HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS = "access-control-expose-headers" + HEADER_EPID_HEADERS = "epid" + HEADER_VERSION_ID = "version-id" + HEADER_COPY_SOURCE_VERSION_ID = "copy-source-version-id" + HEADER_DELETE_MARKER = "delete-marker" + HEADER_WEBSITE_REDIRECT_LOCATION = "website-redirect-location" + HEADER_METADATA_DIRECTIVE = "metadata-directive" + HEADER_EXPIRATION = "expiration" + HEADER_EXPIRES_OBS = "x-obs-expires" + HEADER_RESTORE = "restore" + HEADER_OBJECT_TYPE = "object-type" + HEADER_NEXT_APPEND_POSITION = "next-append-position" + HEADER_STORAGE_CLASS2 = "storage-class" + HEADER_CONTENT_LENGTH = "content-length" + HEADER_CONTENT_TYPE = "content-type" + HEADER_CONTENT_LANGUAGE = "content-language" + HEADER_EXPIRES = "expires" + HEADER_CACHE_CONTROL = "cache-control" + HEADER_CONTENT_DISPOSITION = "content-disposition" + HEADER_CONTENT_ENCODING = "content-encoding" + HEADER_AZ_REDUNDANCY = "az-redundancy" + headerOefMarker = "oef-marker" + + HEADER_ETAG = "etag" + HEADER_LASTMODIFIED = "last-modified" + + HEADER_COPY_SOURCE_IF_MATCH = "copy-source-if-match" + HEADER_COPY_SOURCE_IF_NONE_MATCH = "copy-source-if-none-match" + HEADER_COPY_SOURCE_IF_MODIFIED_SINCE = "copy-source-if-modified-since" + HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE = "copy-source-if-unmodified-since" + + HEADER_IF_MATCH = "If-Match" + HEADER_IF_NONE_MATCH = "If-None-Match" + HEADER_IF_MODIFIED_SINCE = "If-Modified-Since" + HEADER_IF_UNMODIFIED_SINCE = "If-Unmodified-Since" + + HEADER_SSEC_ENCRYPTION = "server-side-encryption-customer-algorithm" + HEADER_SSEC_KEY = "server-side-encryption-customer-key" + HEADER_SSEC_KEY_MD5 = "server-side-encryption-customer-key-MD5" + + HEADER_SSEKMS_ENCRYPTION = "server-side-encryption" + HEADER_SSEKMS_KEY = "server-side-encryption-aws-kms-key-id" + HEADER_SSEKMS_ENCRYPT_KEY_OBS = "server-side-encryption-kms-key-id" + + HEADER_SSEC_COPY_SOURCE_ENCRYPTION = "copy-source-server-side-encryption-customer-algorithm" + HEADER_SSEC_COPY_SOURCE_KEY = "copy-source-server-side-encryption-customer-key" + HEADER_SSEC_COPY_SOURCE_KEY_MD5 = "copy-source-server-side-encryption-customer-key-MD5" + + HEADER_SSEKMS_KEY_AMZ = "x-amz-server-side-encryption-aws-kms-key-id" + + HEADER_SSEKMS_KEY_OBS = "x-obs-server-side-encryption-kms-key-id" + + HEADER_SUCCESS_ACTION_REDIRECT = "success_action_redirect" + + headerFSFileInterface = "fs-file-interface" + + HEADER_DATE_CAMEL = "Date" + HEADER_HOST_CAMEL = "Host" + HEADER_HOST = "host" + HEADER_AUTH_CAMEL = "Authorization" + HEADER_MD5_CAMEL = "Content-MD5" + HEADER_LOCATION_CAMEL = "Location" + HEADER_CONTENT_LENGTH_CAMEL = "Content-Length" + HEADER_CONTENT_TYPE_CAML = "Content-Type" + HEADER_USER_AGENT_CAMEL = "User-Agent" + HEADER_ORIGIN_CAMEL = "Origin" + HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL = "Access-Control-Request-Headers" + HEADER_CACHE_CONTROL_CAMEL = "Cache-Control" + HEADER_CONTENT_DISPOSITION_CAMEL = "Content-Disposition" + HEADER_CONTENT_ENCODING_CAMEL = "Content-Encoding" + HEADER_CONTENT_LANGUAGE_CAMEL = "Content-Language" + HEADER_EXPIRES_CAMEL = "Expires" + + PARAM_VERSION_ID = "versionId" + PARAM_RESPONSE_CONTENT_TYPE = "response-content-type" + PARAM_RESPONSE_CONTENT_LANGUAGE = "response-content-language" + PARAM_RESPONSE_EXPIRES = "response-expires" + PARAM_RESPONSE_CACHE_CONTROL = "response-cache-control" + PARAM_RESPONSE_CONTENT_DISPOSITION = "response-content-disposition" + PARAM_RESPONSE_CONTENT_ENCODING = "response-content-encoding" + PARAM_IMAGE_PROCESS = "x-image-process" + + PARAM_ALGORITHM_AMZ_CAMEL = "X-Amz-Algorithm" + PARAM_CREDENTIAL_AMZ_CAMEL = "X-Amz-Credential" + PARAM_DATE_AMZ_CAMEL = "X-Amz-Date" + PARAM_DATE_OBS_CAMEL = "X-Obs-Date" + PARAM_EXPIRES_AMZ_CAMEL = "X-Amz-Expires" + PARAM_SIGNEDHEADERS_AMZ_CAMEL = "X-Amz-SignedHeaders" + PARAM_SIGNATURE_AMZ_CAMEL = "X-Amz-Signature" + + DEFAULT_SIGNATURE = SignatureV2 + DEFAULT_REGION = "region" + DEFAULT_CONNECT_TIMEOUT = 60 + DEFAULT_SOCKET_TIMEOUT = 60 + DEFAULT_HEADER_TIMEOUT = 60 + DEFAULT_IDLE_CONN_TIMEOUT = 30 + DEFAULT_MAX_RETRY_COUNT = 3 + DEFAULT_MAX_REDIRECT_COUNT = 3 + DEFAULT_MAX_CONN_PER_HOST = 1000 + EMPTY_CONTENT_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD" + LONG_DATE_FORMAT = "20060102T150405Z" + SHORT_DATE_FORMAT = "20060102" + ISO8601_DATE_FORMAT = "2006-01-02T15:04:05Z" + ISO8601_MIDNIGHT_DATE_FORMAT = "2006-01-02T00:00:00Z" + RFC1123_FORMAT = "Mon, 02 Jan 2006 15:04:05 GMT" + + V4_SERVICE_NAME = "s3" + V4_SERVICE_SUFFIX = "aws4_request" + + V2_HASH_PREFIX = "AWS" + OBS_HASH_PREFIX = "OBS" + + V4_HASH_PREFIX = "AWS4-HMAC-SHA256" + V4_HASH_PRE = "AWS4" + + DEFAULT_SSE_KMS_ENCRYPTION = "aws:kms" + DEFAULT_SSE_KMS_ENCRYPTION_OBS = "kms" + + DEFAULT_SSE_C_ENCRYPTION = "AES256" + + HTTP_GET = "GET" + HTTP_POST = "POST" + HTTP_PUT = "PUT" + HTTP_DELETE = "DELETE" + HTTP_HEAD = "HEAD" + HTTP_OPTIONS = "OPTIONS" + + REQUEST_PAYER = "request-payer" + MULTI_AZ = "3az" + + MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 + MIN_PART_SIZE = 100 * 1024 + DEFAULT_PART_SIZE = 9 * 1024 * 1024 + MAX_PART_NUM = 10000 +) + +// SignatureType defines type of signature +type SignatureType string + +const ( + // SignatureV2 signature type v2 + SignatureV2 SignatureType = "v2" + // SignatureV4 signature type v4 + SignatureV4 SignatureType = "v4" + // SignatureObs signature type OBS + SignatureObs SignatureType = "OBS" +) + +var ( + interestedHeaders = []string{"content-md5", "content-type", "date"} + + allowedRequestHTTPHeaderMetadataNames = map[string]bool{ + "content-type": true, + "content-md5": true, + "content-length": true, + "content-language": true, + "expires": true, + "origin": true, + "cache-control": true, + "content-disposition": true, + "content-encoding": true, + "access-control-request-method": true, + "access-control-request-headers": true, + "x-default-storage-class": true, + "location": true, + "date": true, + "etag": true, + "range": true, + "host": true, + "if-modified-since": true, + "if-unmodified-since": true, + "if-match": true, + "if-none-match": true, + "last-modified": true, + "content-range": true, + } + + allowedResourceParameterNames = map[string]bool{ + "acl": true, + "backtosource": true, + "metadata": true, + "policy": true, + "torrent": true, + "logging": true, + "location": true, + "storageinfo": true, + "quota": true, + "storageclass": true, + "storagepolicy": true, + "requestpayment": true, + "versions": true, + "versioning": true, + "versionid": true, + "uploads": true, + "uploadid": true, + "partnumber": true, + "website": true, + "notification": true, + "lifecycle": true, + "deletebucket": true, + "delete": true, + "cors": true, + "restore": true, + "tagging": true, + "append": true, + "position": true, + "replication": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "x-image-process": true, + "x-oss-process": true, + "x-image-save-bucket": true, + "x-image-save-object": true, + "ignore-sign-in-query": true, + } + + mimeTypes = map[string]string{ + "001": "application/x-001", + "301": "application/x-301", + "323": "text/h323", + "7z": "application/x-7z-compressed", + "906": "application/x-906", + "907": "drawing/907", + "IVF": "video/x-ivf", + "a11": "application/x-a11", + "aac": "audio/x-aac", + "acp": "audio/x-mei-aac", + "ai": "application/postscript", + "aif": "audio/aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "anv": "application/x-anv", + "apk": "application/vnd.android.package-archive", + "asa": "text/asa", + "asf": "video/x-ms-asf", + "asp": "text/asp", + "asx": "video/x-ms-asf", + "atom": "application/atom+xml", + "au": "audio/basic", + "avi": "video/avi", + "awf": "application/vnd.adobe.workflow", + "biz": "text/xml", + "bmp": "application/x-bmp", + "bot": "application/x-bot", + "bz2": "application/x-bzip2", + "c4t": "application/x-c4t", + "c90": "application/x-c90", + "cal": "application/x-cals", + "cat": "application/vnd.ms-pki.seccat", + "cdf": "application/x-netcdf", + "cdr": "application/x-cdr", + "cel": "application/x-cel", + "cer": "application/x-x509-ca-cert", + "cg4": "application/x-g4", + "cgm": "application/x-cgm", + "cit": "application/x-cit", + "class": "java/*", + "cml": "text/xml", + "cmp": "application/x-cmp", + "cmx": "application/x-cmx", + "cot": "application/x-cot", + "crl": "application/pkix-crl", + "crt": "application/x-x509-ca-cert", + "csi": "application/x-csi", + "css": "text/css", + "csv": "text/csv", + "cu": "application/cu-seeme", + "cut": "application/x-cut", + "dbf": "application/x-dbf", + "dbm": "application/x-dbm", + "dbx": "application/x-dbx", + "dcd": "text/xml", + "dcx": "application/x-dcx", + "deb": "application/x-debian-package", + "der": "application/x-x509-ca-cert", + "dgn": "application/x-dgn", + "dib": "application/x-dib", + "dll": "application/x-msdownload", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dot": "application/msword", + "drw": "application/x-drw", + "dtd": "text/xml", + "dvi": "application/x-dvi", + "dwf": "application/x-dwf", + "dwg": "application/x-dwg", + "dxb": "application/x-dxb", + "dxf": "application/x-dxf", + "edn": "application/vnd.adobe.edn", + "emf": "application/x-emf", + "eml": "message/rfc822", + "ent": "text/xml", + "eot": "application/vnd.ms-fontobject", + "epi": "application/x-epi", + "eps": "application/postscript", + "epub": "application/epub+zip", + "etd": "application/x-ebx", + "etx": "text/x-setext", + "exe": "application/x-msdownload", + "fax": "image/fax", + "fdf": "application/vnd.fdf", + "fif": "application/fractals", + "flac": "audio/flac", + "flv": "video/x-flv", + "fo": "text/xml", + "frm": "application/x-frm", + "g4": "application/x-g4", + "gbr": "application/x-gbr", + "gif": "image/gif", + "gl2": "application/x-gl2", + "gp4": "application/x-gp4", + "gz": "application/gzip", + "hgl": "application/x-hgl", + "hmr": "application/x-hmr", + "hpg": "application/x-hpgl", + "hpl": "application/x-hpl", + "hqx": "application/mac-binhex40", + "hrf": "application/x-hrf", + "hta": "application/hta", + "htc": "text/x-component", + "htm": "text/html", + "html": "text/html", + "htt": "text/webviewhtml", + "htx": "text/html", + "icb": "application/x-icb", + "ico": "application/x-ico", + "ics": "text/calendar", + "iff": "application/x-iff", + "ig4": "application/x-g4", + "igs": "application/x-igs", + "iii": "application/x-iphone", + "img": "application/x-img", + "ini": "text/plain", + "ins": "application/x-internet-signup", + "ipa": "application/vnd.iphone", + "iso": "application/x-iso9660-image", + "isp": "application/x-internet-signup", + "jar": "application/java-archive", + "java": "java/*", + "jfif": "image/jpeg", + "jpe": "image/jpeg", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "js": "application/x-javascript", + "json": "application/json", + "jsp": "text/html", + "la1": "audio/x-liquid-file", + "lar": "application/x-laplayer-reg", + "latex": "application/x-latex", + "lavs": "audio/x-liquid-secure", + "lbm": "application/x-lbm", + "lmsff": "audio/x-la-lms", + "log": "text/plain", + "ls": "application/x-javascript", + "ltr": "application/x-ltr", + "m1v": "video/x-mpeg", + "m2v": "video/x-mpeg", + "m3u": "audio/mpegurl", + "m4a": "audio/mp4", + "m4e": "video/mpeg4", + "m4v": "video/mp4", + "mac": "application/x-mac", + "man": "application/x-troff-man", + "math": "text/xml", + "mdb": "application/msaccess", + "mfp": "application/x-shockwave-flash", + "mht": "message/rfc822", + "mhtml": "message/rfc822", + "mi": "application/x-mi", + "mid": "audio/mid", + "midi": "audio/mid", + "mil": "application/x-mil", + "mml": "text/xml", + "mnd": "audio/x-musicnet-download", + "mns": "audio/x-musicnet-stream", + "mocha": "application/x-javascript", + "mov": "video/quicktime", + "movie": "video/x-sgi-movie", + "mp1": "audio/mp1", + "mp2": "audio/mp2", + "mp2v": "video/mpeg", + "mp3": "audio/mp3", + "mp4": "video/mp4", + "mp4a": "audio/mp4", + "mp4v": "video/mp4", + "mpa": "video/x-mpg", + "mpd": "application/vnd.ms-project", + "mpe": "video/mpeg", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpg4": "video/mp4", + "mpga": "audio/rn-mpeg", + "mpp": "application/vnd.ms-project", + "mps": "video/x-mpeg", + "mpt": "application/vnd.ms-project", + "mpv": "video/mpg", + "mpv2": "video/mpeg", + "mpw": "application/vnd.ms-project", + "mpx": "application/vnd.ms-project", + "mtx": "text/xml", + "mxp": "application/x-mmxp", + "net": "image/pnetvue", + "nrf": "application/x-nrf", + "nws": "message/rfc822", + "odc": "text/x-ms-odc", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "ogv": "video/ogg", + "ogx": "application/ogg", + "out": "application/x-out", + "p10": "application/pkcs10", + "p12": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "p7c": "application/pkcs7-mime", + "p7m": "application/pkcs7-mime", + "p7r": "application/x-pkcs7-certreqresp", + "p7s": "application/pkcs7-signature", + "pbm": "image/x-portable-bitmap", + "pc5": "application/x-pc5", + "pci": "application/x-pci", + "pcl": "application/x-pcl", + "pcx": "application/x-pcx", + "pdf": "application/pdf", + "pdx": "application/vnd.adobe.pdx", + "pfx": "application/x-pkcs12", + "pgl": "application/x-pgl", + "pgm": "image/x-portable-graymap", + "pic": "application/x-pic", + "pko": "application/vnd.ms-pki.pko", + "pl": "application/x-perl", + "plg": "text/html", + "pls": "audio/scpls", + "plt": "application/x-plt", + "png": "image/png", + "pnm": "image/x-portable-anymap", + "pot": "application/vnd.ms-powerpoint", + "ppa": "application/vnd.ms-powerpoint", + "ppm": "application/x-ppm", + "pps": "application/vnd.ms-powerpoint", + "ppt": "application/vnd.ms-powerpoint", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pr": "application/x-pr", + "prf": "application/pics-rules", + "prn": "application/x-prn", + "prt": "application/x-prt", + "ps": "application/postscript", + "ptn": "application/x-ptn", + "pwz": "application/vnd.ms-powerpoint", + "qt": "video/quicktime", + "r3t": "text/vnd.rn-realtext3d", + "ra": "audio/vnd.rn-realaudio", + "ram": "audio/x-pn-realaudio", + "rar": "application/x-rar-compressed", + "ras": "application/x-ras", + "rat": "application/rat-file", + "rdf": "text/xml", + "rec": "application/vnd.rn-recording", + "red": "application/x-red", + "rgb": "application/x-rgb", + "rjs": "application/vnd.rn-realsystem-rjs", + "rjt": "application/vnd.rn-realsystem-rjt", + "rlc": "application/x-rlc", + "rle": "application/x-rle", + "rm": "application/vnd.rn-realmedia", + "rmf": "application/vnd.adobe.rmf", + "rmi": "audio/mid", + "rmj": "application/vnd.rn-realsystem-rmj", + "rmm": "audio/x-pn-realaudio", + "rmp": "application/vnd.rn-rn_music_package", + "rms": "application/vnd.rn-realmedia-secure", + "rmvb": "application/vnd.rn-realmedia-vbr", + "rmx": "application/vnd.rn-realsystem-rmx", + "rnx": "application/vnd.rn-realplayer", + "rp": "image/vnd.rn-realpix", + "rpm": "audio/x-pn-realaudio-plugin", + "rsml": "application/vnd.rn-rsml", + "rss": "application/rss+xml", + "rt": "text/vnd.rn-realtext", + "rtf": "application/x-rtf", + "rv": "video/vnd.rn-realvideo", + "sam": "application/x-sam", + "sat": "application/x-sat", + "sdp": "application/sdp", + "sdw": "application/x-sdw", + "sgm": "text/sgml", + "sgml": "text/sgml", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "sit": "application/x-stuffit", + "slb": "application/x-slb", + "sld": "application/x-sld", + "slk": "drawing/x-slk", + "smi": "application/smil", + "smil": "application/smil", + "smk": "application/x-smk", + "snd": "audio/basic", + "sol": "text/plain", + "sor": "text/plain", + "spc": "application/x-pkcs7-certificates", + "spl": "application/futuresplash", + "spp": "text/xml", + "ssm": "application/streamingmedia", + "sst": "application/vnd.ms-pki.certstore", + "stl": "application/vnd.ms-pki.stl", + "stm": "text/html", + "sty": "application/x-sty", + "svg": "image/svg+xml", + "swf": "application/x-shockwave-flash", + "tar": "application/x-tar", + "tdf": "application/x-tdf", + "tg4": "application/x-tg4", + "tga": "application/x-tga", + "tif": "image/tiff", + "tiff": "image/tiff", + "tld": "text/xml", + "top": "drawing/x-top", + "torrent": "application/x-bittorrent", + "tsd": "text/xml", + "ttf": "application/x-font-ttf", + "txt": "text/plain", + "uin": "application/x-icq", + "uls": "text/iuls", + "vcf": "text/x-vcard", + "vda": "application/x-vda", + "vdx": "application/vnd.visio", + "vml": "text/xml", + "vpg": "application/x-vpeg005", + "vsd": "application/vnd.visio", + "vss": "application/vnd.visio", + "vst": "application/x-vst", + "vsw": "application/vnd.visio", + "vsx": "application/vnd.visio", + "vtx": "application/vnd.visio", + "vxml": "text/xml", + "wav": "audio/wav", + "wax": "audio/x-ms-wax", + "wb1": "application/x-wb1", + "wb2": "application/x-wb2", + "wb3": "application/x-wb3", + "wbmp": "image/vnd.wap.wbmp", + "webm": "video/webm", + "wiz": "application/msword", + "wk3": "application/x-wk3", + "wk4": "application/x-wk4", + "wkq": "application/x-wkq", + "wks": "application/x-wks", + "wm": "video/x-ms-wm", + "wma": "audio/x-ms-wma", + "wmd": "application/x-ms-wmd", + "wmf": "application/x-wmf", + "wml": "text/vnd.wap.wml", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wmz": "application/x-ms-wmz", + "woff": "application/x-font-woff", + "wp6": "application/x-wp6", + "wpd": "application/x-wpd", + "wpg": "application/x-wpg", + "wpl": "application/vnd.ms-wpl", + "wq1": "application/x-wq1", + "wr1": "application/x-wr1", + "wri": "application/x-wri", + "wrk": "application/x-wrk", + "ws": "application/x-ws", + "ws2": "application/x-ws", + "wsc": "text/scriptlet", + "wsdl": "text/xml", + "wvx": "video/x-ms-wvx", + "x_b": "application/x-x_b", + "x_t": "application/x-x_t", + "xap": "application/x-silverlight-app", + "xbm": "image/x-xbitmap", + "xdp": "application/vnd.adobe.xdp", + "xdr": "text/xml", + "xfd": "application/vnd.adobe.xfd", + "xfdf": "application/vnd.adobe.xfdf", + "xhtml": "text/html", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlw": "application/x-xlw", + "xml": "text/xml", + "xpl": "audio/scpls", + "xpm": "image/x-xpixmap", + "xq": "text/xml", + "xql": "text/xml", + "xquery": "text/xml", + "xsd": "text/xml", + "xsl": "text/xml", + "xslt": "text/xml", + "xwd": "application/x-xwd", + "yaml": "text/yaml", + "yml": "text/yaml", + "zip": "application/zip", + } +) + +// HttpMethodType defines http method type +type HttpMethodType string + +const ( + HttpMethodGet HttpMethodType = HTTP_GET + HttpMethodPut HttpMethodType = HTTP_PUT + HttpMethodPost HttpMethodType = HTTP_POST + HttpMethodDelete HttpMethodType = HTTP_DELETE + HttpMethodHead HttpMethodType = HTTP_HEAD + HttpMethodOptions HttpMethodType = HTTP_OPTIONS +) + +// SubResourceType defines the subResource value +type SubResourceType string + +const ( + // SubResourceStoragePolicy subResource value: storagePolicy + SubResourceStoragePolicy SubResourceType = "storagePolicy" + + // SubResourceStorageClass subResource value: storageClass + SubResourceStorageClass SubResourceType = "storageClass" + + // SubResourceQuota subResource value: quota + SubResourceQuota SubResourceType = "quota" + + // SubResourceStorageInfo subResource value: storageinfo + SubResourceStorageInfo SubResourceType = "storageinfo" + + // SubResourceLocation subResource value: location + SubResourceLocation SubResourceType = "location" + + // SubResourceAcl subResource value: acl + SubResourceAcl SubResourceType = "acl" + + // SubResourcePolicy subResource value: policy + SubResourcePolicy SubResourceType = "policy" + + // SubResourceCors subResource value: cors + SubResourceCors SubResourceType = "cors" + + // SubResourceVersioning subResource value: versioning + SubResourceVersioning SubResourceType = "versioning" + + // SubResourceWebsite subResource value: website + SubResourceWebsite SubResourceType = "website" + + // SubResourceLogging subResource value: logging + SubResourceLogging SubResourceType = "logging" + + // SubResourceLifecycle subResource value: lifecycle + SubResourceLifecycle SubResourceType = "lifecycle" + + // SubResourceNotification subResource value: notification + SubResourceNotification SubResourceType = "notification" + + // SubResourceTagging subResource value: tagging + SubResourceTagging SubResourceType = "tagging" + + // SubResourceDelete subResource value: delete + SubResourceDelete SubResourceType = "delete" + + // SubResourceVersions subResource value: versions + SubResourceVersions SubResourceType = "versions" + + // SubResourceUploads subResource value: uploads + SubResourceUploads SubResourceType = "uploads" + + // SubResourceRestore subResource value: restore + SubResourceRestore SubResourceType = "restore" + + // SubResourceMetadata subResource value: metadata + SubResourceMetadata SubResourceType = "metadata" + + // SubResourceRequestPayment subResource value: requestPayment + SubResourceRequestPayment SubResourceType = "requestPayment" +) + +// objectKeyType defines the objectKey value +type objectKeyType string + +const ( + // objectKeyExtensionPolicy objectKey value: v1/extension_policy + objectKeyExtensionPolicy objectKeyType = "v1/extension_policy" + + // objectKeyAsyncFetchJob objectKey value: v1/async-fetch/jobs + objectKeyAsyncFetchJob objectKeyType = "v1/async-fetch/jobs" +) + +// AclType defines bucket/object acl type +type AclType string + +const ( + AclPrivate AclType = "private" + AclPublicRead AclType = "public-read" + AclPublicReadWrite AclType = "public-read-write" + AclAuthenticatedRead AclType = "authenticated-read" + AclBucketOwnerRead AclType = "bucket-owner-read" + AclBucketOwnerFullControl AclType = "bucket-owner-full-control" + AclLogDeliveryWrite AclType = "log-delivery-write" + AclPublicReadDelivery AclType = "public-read-delivered" + AclPublicReadWriteDelivery AclType = "public-read-write-delivered" +) + +// StorageClassType defines bucket storage class +type StorageClassType string + +const ( + //StorageClassStandard storage class: STANDARD + StorageClassStandard StorageClassType = "STANDARD" + + //StorageClassWarm storage class: WARM + StorageClassWarm StorageClassType = "WARM" + + //StorageClassCold storage class: COLD + StorageClassCold StorageClassType = "COLD" + + storageClassStandardIA StorageClassType = "STANDARD_IA" + storageClassGlacier StorageClassType = "GLACIER" +) + +// PermissionType defines permission type +type PermissionType string + +const ( + // PermissionRead permission type: READ + PermissionRead PermissionType = "READ" + + // PermissionWrite permission type: WRITE + PermissionWrite PermissionType = "WRITE" + + // PermissionReadAcp permission type: READ_ACP + PermissionReadAcp PermissionType = "READ_ACP" + + // PermissionWriteAcp permission type: WRITE_ACP + PermissionWriteAcp PermissionType = "WRITE_ACP" + + // PermissionFullControl permission type: FULL_CONTROL + PermissionFullControl PermissionType = "FULL_CONTROL" +) + +// GranteeType defines grantee type +type GranteeType string + +const ( + // GranteeGroup grantee type: Group + GranteeGroup GranteeType = "Group" + + // GranteeUser grantee type: CanonicalUser + GranteeUser GranteeType = "CanonicalUser" +) + +// GroupUriType defines grantee uri type +type GroupUriType string + +const ( + // GroupAllUsers grantee uri type: AllUsers + GroupAllUsers GroupUriType = "AllUsers" + + // GroupAuthenticatedUsers grantee uri type: AuthenticatedUsers + GroupAuthenticatedUsers GroupUriType = "AuthenticatedUsers" + + // GroupLogDelivery grantee uri type: LogDelivery + GroupLogDelivery GroupUriType = "LogDelivery" +) + +// VersioningStatusType defines bucket version status +type VersioningStatusType string + +const ( + // VersioningStatusEnabled version status: Enabled + VersioningStatusEnabled VersioningStatusType = "Enabled" + + // VersioningStatusSuspended version status: Suspended + VersioningStatusSuspended VersioningStatusType = "Suspended" +) + +// ProtocolType defines protocol type +type ProtocolType string + +const ( + // ProtocolHttp prorocol type: http + ProtocolHttp ProtocolType = "http" + + // ProtocolHttps prorocol type: https + ProtocolHttps ProtocolType = "https" +) + +// RuleStatusType defines lifeCycle rule status +type RuleStatusType string + +const ( + // RuleStatusEnabled rule status: Enabled + RuleStatusEnabled RuleStatusType = "Enabled" + + // RuleStatusDisabled rule status: Disabled + RuleStatusDisabled RuleStatusType = "Disabled" +) + +// RestoreTierType defines restore options +type RestoreTierType string + +const ( + // RestoreTierExpedited restore options: Expedited + RestoreTierExpedited RestoreTierType = "Expedited" + + // RestoreTierStandard restore options: Standard + RestoreTierStandard RestoreTierType = "Standard" + + // RestoreTierBulk restore options: Bulk + RestoreTierBulk RestoreTierType = "Bulk" +) + +// MetadataDirectiveType defines metadata operation indicator +type MetadataDirectiveType string + +const ( + // CopyMetadata metadata operation: COPY + CopyMetadata MetadataDirectiveType = "COPY" + + // ReplaceNew metadata operation: REPLACE_NEW + ReplaceNew MetadataDirectiveType = "REPLACE_NEW" + + // ReplaceMetadata metadata operation: REPLACE + ReplaceMetadata MetadataDirectiveType = "REPLACE" +) + +// EventType defines bucket notification type of events +type EventType string + +const ( + // ObjectCreatedAll type of events: ObjectCreated:* + ObjectCreatedAll EventType = "ObjectCreated:*" + + // ObjectCreatedPut type of events: ObjectCreated:Put + ObjectCreatedPut EventType = "ObjectCreated:Put" + + // ObjectCreatedPost type of events: ObjectCreated:Post + ObjectCreatedPost EventType = "ObjectCreated:Post" + + // ObjectCreatedCopy type of events: ObjectCreated:Copy + ObjectCreatedCopy EventType = "ObjectCreated:Copy" + + // ObjectCreatedCompleteMultipartUpload type of events: ObjectCreated:CompleteMultipartUpload + ObjectCreatedCompleteMultipartUpload EventType = "ObjectCreated:CompleteMultipartUpload" + + // ObjectRemovedAll type of events: ObjectRemoved:* + ObjectRemovedAll EventType = "ObjectRemoved:*" + + // ObjectRemovedDelete type of events: ObjectRemoved:Delete + ObjectRemovedDelete EventType = "ObjectRemoved:Delete" + + // ObjectRemovedDeleteMarkerCreated type of events: ObjectRemoved:DeleteMarkerCreated + ObjectRemovedDeleteMarkerCreated EventType = "ObjectRemoved:DeleteMarkerCreated" +) + +// PayerType defines type of payer +type PayerType string + +const ( + // BucketOwnerPayer type of payer: BucketOwner + BucketOwnerPayer PayerType = "BucketOwner" + + // RequesterPayer type of payer: Requester + RequesterPayer PayerType = "Requester" + + // Requester header for requester-Pays + Requester PayerType = "requester" +) + +// FetchPolicyStatusType defines type of fetch policy status +type FetchPolicyStatusType string + +const ( + // FetchStatusOpen type of status: open + FetchStatusOpen FetchPolicyStatusType = "open" + + // FetchStatusClosed type of status: closed + FetchStatusClosed FetchPolicyStatusType = "closed" +) + +// AvailableZoneType defines type of az redundancy +type AvailableZoneType string + +const ( + AvailableZoneMultiAz AvailableZoneType = "3az" +) + +// FSStatusType defines type of file system status +type FSStatusType string + +const ( + FSStatusEnabled FSStatusType = "Enabled" + FSStatusDisabled FSStatusType = "Disabled" +) diff --git a/cve-vulner-manager/obs/convert.go b/cve-vulner-manager/obs/convert.go new file mode 100644 index 0000000000000000000000000000000000000000..6ed04ac84a7176066aa2ff88662ae786c9311935 --- /dev/null +++ b/cve-vulner-manager/obs/convert.go @@ -0,0 +1,896 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "reflect" + "strings" + "time" +) + +func cleanHeaderPrefix(header http.Header) map[string][]string { + responseHeaders := make(map[string][]string) + for key, value := range header { + if len(value) > 0 { + key = strings.ToLower(key) + if strings.HasPrefix(key, HEADER_PREFIX) || strings.HasPrefix(key, HEADER_PREFIX_OBS) { + key = key[len(HEADER_PREFIX):] + } + responseHeaders[key] = value + } + } + return responseHeaders +} + +// ParseStringToEventType converts string value to EventType value and returns it +func ParseStringToEventType(value string) (ret EventType) { + switch value { + case "ObjectCreated:*", "s3:ObjectCreated:*": + ret = ObjectCreatedAll + case "ObjectCreated:Put", "s3:ObjectCreated:Put": + ret = ObjectCreatedPut + case "ObjectCreated:Post", "s3:ObjectCreated:Post": + ret = ObjectCreatedPost + case "ObjectCreated:Copy", "s3:ObjectCreated:Copy": + ret = ObjectCreatedCopy + case "ObjectCreated:CompleteMultipartUpload", "s3:ObjectCreated:CompleteMultipartUpload": + ret = ObjectCreatedCompleteMultipartUpload + case "ObjectRemoved:*", "s3:ObjectRemoved:*": + ret = ObjectRemovedAll + case "ObjectRemoved:Delete", "s3:ObjectRemoved:Delete": + ret = ObjectRemovedDelete + case "ObjectRemoved:DeleteMarkerCreated", "s3:ObjectRemoved:DeleteMarkerCreated": + ret = ObjectRemovedDeleteMarkerCreated + default: + ret = "" + } + return +} + +// ParseStringToStorageClassType converts string value to StorageClassType value and returns it +func ParseStringToStorageClassType(value string) (ret StorageClassType) { + switch value { + case "STANDARD": + ret = StorageClassStandard + case "STANDARD_IA", "WARM": + ret = StorageClassWarm + case "GLACIER", "COLD": + ret = StorageClassCold + default: + ret = "" + } + return +} + +func prepareGrantURI(grant Grant) string { + if grant.Grantee.URI == GroupAllUsers || grant.Grantee.URI == GroupAuthenticatedUsers { + return fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/global/", grant.Grantee.URI) + } + if grant.Grantee.URI == GroupLogDelivery { + return fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/s3/", grant.Grantee.URI) + } + return fmt.Sprintf("%s", grant.Grantee.URI) +} + +func convertGrantToXML(grant Grant, isObs bool, isBucket bool) string { + xml := make([]string, 0, 4) + + if grant.Grantee.Type == GranteeUser { + if isObs { + xml = append(xml, "") + } else { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + } + if grant.Grantee.ID != "" { + granteeID := XmlTranscoding(grant.Grantee.ID) + xml = append(xml, fmt.Sprintf("%s", granteeID)) + } + if !isObs && grant.Grantee.DisplayName != "" { + granteeDisplayName := XmlTranscoding(grant.Grantee.DisplayName) + xml = append(xml, fmt.Sprintf("%s", granteeDisplayName)) + } + xml = append(xml, "") + } else { + if !isObs { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + xml = append(xml, prepareGrantURI(grant)) + xml = append(xml, "") + } else if grant.Grantee.URI == GroupAllUsers { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("Everyone")) + xml = append(xml, "") + } else { + return strings.Join(xml, "") + } + } + + xml = append(xml, fmt.Sprintf("%s", grant.Permission)) + if isObs && isBucket { + xml = append(xml, fmt.Sprintf("%t", grant.Delivered)) + } + xml = append(xml, fmt.Sprintf("")) + return strings.Join(xml, "") +} + +func hasLoggingTarget(input BucketLoggingStatus) bool { + if input.TargetBucket != "" || input.TargetPrefix != "" || len(input.TargetGrants) > 0 { + return true + } + return false +} + +// ConvertLoggingStatusToXml converts BucketLoggingStatus value to XML data and returns it +func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool, isObs bool) (data string, md5 string) { + grantsLength := len(input.TargetGrants) + xml := make([]string, 0, 8+grantsLength) + + xml = append(xml, "") + if isObs && input.Agency != "" { + agency := XmlTranscoding(input.Agency) + xml = append(xml, fmt.Sprintf("%s", agency)) + } + if hasLoggingTarget(input) { + xml = append(xml, "") + if input.TargetBucket != "" { + xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) + } + if input.TargetPrefix != "" { + targetPrefix := XmlTranscoding(input.TargetPrefix) + xml = append(xml, fmt.Sprintf("%s", targetPrefix)) + } + if grantsLength > 0 { + xml = append(xml, "") + for _, grant := range input.TargetGrants { + xml = append(xml, convertGrantToXML(grant, isObs, false)) + } + xml = append(xml, "") + } + + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +// ConvertAclToXml converts AccessControlPolicy value to XML data and returns it +func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + if isObs && input.Delivered != "" { + objectDelivered := XmlTranscoding(input.Delivered) + xml = append(xml, fmt.Sprintf("%s", objectDelivered)) + } else { + xml = append(xml, "") + } + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXML(grant, isObs, false)) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertBucketACLToXML(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + + xml = append(xml, "") + + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXML(grant, isObs, true)) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertConditionToXML(condition Condition) string { + xml := make([]string, 0, 2) + if condition.KeyPrefixEquals != "" { + keyPrefixEquals := XmlTranscoding(condition.KeyPrefixEquals) + xml = append(xml, fmt.Sprintf("%s", keyPrefixEquals)) + } + if condition.HttpErrorCodeReturnedEquals != "" { + xml = append(xml, fmt.Sprintf("%s", condition.HttpErrorCodeReturnedEquals)) + } + if len(xml) > 0 { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return "" +} + +func prepareRoutingRule(input BucketWebsiteConfiguration) string { + xml := make([]string, 0, len(input.RoutingRules)*10) + for _, routingRule := range input.RoutingRules { + xml = append(xml, "") + xml = append(xml, "") + if routingRule.Redirect.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.Protocol)) + } + if routingRule.Redirect.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HostName)) + } + if routingRule.Redirect.ReplaceKeyPrefixWith != "" { + replaceKeyPrefixWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyPrefixWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyPrefixWith)) + } + + if routingRule.Redirect.ReplaceKeyWith != "" { + replaceKeyWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyWith)) + } + if routingRule.Redirect.HttpRedirectCode != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HttpRedirectCode)) + } + xml = append(xml, "") + + if ret := convertConditionToXML(routingRule.Condition); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + return strings.Join(xml, "") +} + +// ConvertWebsiteConfigurationToXml converts BucketWebsiteConfiguration value to XML data and returns it +func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd5 bool) (data string, md5 string) { + routingRuleLength := len(input.RoutingRules) + xml := make([]string, 0, 6+routingRuleLength*10) + xml = append(xml, "") + + if input.RedirectAllRequestsTo.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", input.RedirectAllRequestsTo.HostName)) + if input.RedirectAllRequestsTo.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", input.RedirectAllRequestsTo.Protocol)) + } + xml = append(xml, "") + } else { + if input.IndexDocument.Suffix != "" { + indexDocumentSuffix := XmlTranscoding(input.IndexDocument.Suffix) + xml = append(xml, fmt.Sprintf("%s", indexDocumentSuffix)) + } + if input.ErrorDocument.Key != "" { + errorDocumentKey := XmlTranscoding(input.ErrorDocument.Key) + xml = append(xml, fmt.Sprintf("%s", errorDocumentKey)) + } + if routingRuleLength > 0 { + xml = append(xml, "") + xml = append(xml, prepareRoutingRule(input)) + xml = append(xml, "") + } + } + + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertTransitionsToXML(transitions []Transition, isObs bool) string { + if length := len(transitions); length > 0 { + xml := make([]string, 0, length) + for _, transition := range transitions { + var temp string + if transition.Days > 0 { + temp = fmt.Sprintf("%d", transition.Days) + } else if !transition.Date.IsZero() { + temp = fmt.Sprintf("%s", transition.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) + } + if temp != "" { + if !isObs { + storageClass := string(transition.StorageClass) + if transition.StorageClass == StorageClassWarm { + storageClass = string(storageClassStandardIA) + } else if transition.StorageClass == StorageClassCold { + storageClass = string(storageClassGlacier) + } + xml = append(xml, fmt.Sprintf("%s%s", temp, storageClass)) + } else { + xml = append(xml, fmt.Sprintf("%s%s", temp, transition.StorageClass)) + } + } + } + return strings.Join(xml, "") + } + return "" +} + +func convertExpirationToXML(expiration Expiration) string { + if expiration.Days > 0 { + return fmt.Sprintf("%d", expiration.Days) + } else if !expiration.Date.IsZero() { + return fmt.Sprintf("%s", expiration.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) + } + return "" +} +func convertNoncurrentVersionTransitionsToXML(noncurrentVersionTransitions []NoncurrentVersionTransition, isObs bool) string { + if length := len(noncurrentVersionTransitions); length > 0 { + xml := make([]string, 0, length) + for _, noncurrentVersionTransition := range noncurrentVersionTransitions { + if noncurrentVersionTransition.NoncurrentDays > 0 { + storageClass := string(noncurrentVersionTransition.StorageClass) + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + xml = append(xml, fmt.Sprintf("%d"+ + "%s", + noncurrentVersionTransition.NoncurrentDays, storageClass)) + } + } + return strings.Join(xml, "") + } + return "" +} +func convertNoncurrentVersionExpirationToXML(noncurrentVersionExpiration NoncurrentVersionExpiration) string { + if noncurrentVersionExpiration.NoncurrentDays > 0 { + return fmt.Sprintf("%d", noncurrentVersionExpiration.NoncurrentDays) + } + return "" +} + +// ConvertLifecyleConfigurationToXml converts BucketLifecyleConfiguration value to XML data and returns it +func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.LifecycleRules)*9) + xml = append(xml, "") + for _, lifecyleRule := range input.LifecycleRules { + xml = append(xml, "") + if lifecyleRule.ID != "" { + lifecyleRuleID := XmlTranscoding(lifecyleRule.ID) + xml = append(xml, fmt.Sprintf("%s", lifecyleRuleID)) + } + lifecyleRulePrefix := XmlTranscoding(lifecyleRule.Prefix) + xml = append(xml, fmt.Sprintf("%s", lifecyleRulePrefix)) + xml = append(xml, fmt.Sprintf("%s", lifecyleRule.Status)) + if ret := convertTransitionsToXML(lifecyleRule.Transitions, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := convertExpirationToXML(lifecyleRule.Expiration); ret != "" { + xml = append(xml, ret) + } + if ret := convertNoncurrentVersionTransitionsToXML(lifecyleRule.NoncurrentVersionTransitions, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := convertNoncurrentVersionExpirationToXML(lifecyleRule.NoncurrentVersionExpiration); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func converntFilterRulesToXML(filterRules []FilterRule, isObs bool) string { + if length := len(filterRules); length > 0 { + xml := make([]string, 0, length*4) + for _, filterRule := range filterRules { + xml = append(xml, "") + if filterRule.Name != "" { + filterRuleName := XmlTranscoding(filterRule.Name) + xml = append(xml, fmt.Sprintf("%s", filterRuleName)) + } + if filterRule.Value != "" { + filterRuleValue := XmlTranscoding(filterRule.Value) + xml = append(xml, fmt.Sprintf("%s", filterRuleValue)) + } + xml = append(xml, "") + } + if !isObs { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return "" +} + +func converntEventsToXML(events []EventType, isObs bool) string { + if length := len(events); length > 0 { + xml := make([]string, 0, length) + if !isObs { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s%s", "s3:", event)) + } + } else { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s", event)) + } + } + return strings.Join(xml, "") + } + return "" +} + +func converntConfigureToXML(topicConfiguration TopicConfiguration, xmlElem string, isObs bool) string { + xml := make([]string, 0, 6) + xml = append(xml, xmlElem) + if topicConfiguration.ID != "" { + topicConfigurationID := XmlTranscoding(topicConfiguration.ID) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationID)) + } + topicConfigurationTopic := XmlTranscoding(topicConfiguration.Topic) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationTopic)) + + if ret := converntEventsToXML(topicConfiguration.Events, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := converntFilterRulesToXML(topicConfiguration.FilterRules, isObs); ret != "" { + xml = append(xml, ret) + } + tempElem := xmlElem[0:1] + "/" + xmlElem[1:] + xml = append(xml, tempElem) + return strings.Join(xml, "") +} + +// ConverntObsRestoreToXml converts RestoreObjectInput value to XML data and returns it +func ConverntObsRestoreToXml(restoreObjectInput RestoreObjectInput) string { + xml := make([]string, 0, 2) + xml = append(xml, fmt.Sprintf("%d", restoreObjectInput.Days)) + if restoreObjectInput.Tier != "Bulk" { + xml = append(xml, fmt.Sprintf("%s", restoreObjectInput.Tier)) + } + xml = append(xml, fmt.Sprintf("")) + data := strings.Join(xml, "") + return data +} + +// ConvertNotificationToXml converts BucketNotification value to XML data and returns it +func ConvertNotificationToXml(input BucketNotification, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.TopicConfigurations)*6) + xml = append(xml, "") + for _, topicConfiguration := range input.TopicConfigurations { + ret := converntConfigureToXML(topicConfiguration, "", isObs) + xml = append(xml, ret) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +// ConvertCompleteMultipartUploadInputToXml converts CompleteMultipartUploadInput value to XML data and returns it +func ConvertCompleteMultipartUploadInputToXml(input CompleteMultipartUploadInput, returnMd5 bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.Parts)*4) + xml = append(xml, "") + for _, part := range input.Parts { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("%d", part.PartNumber)) + xml = append(xml, fmt.Sprintf("%s", part.ETag)) + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func parseSseHeader(responseHeaders map[string][]string) (sseHeader ISseHeader) { + if ret, ok := responseHeaders[HEADER_SSEC_ENCRYPTION]; ok { + sseCHeader := SseCHeader{Encryption: ret[0]} + if ret, ok = responseHeaders[HEADER_SSEC_KEY_MD5]; ok { + sseCHeader.KeyMD5 = ret[0] + } + sseHeader = sseCHeader + } else if ret, ok := responseHeaders[HEADER_SSEKMS_ENCRYPTION]; ok { + sseKmsHeader := SseKmsHeader{Encryption: ret[0]} + if ret, ok = responseHeaders[HEADER_SSEKMS_KEY]; ok { + sseKmsHeader.Key = ret[0] + } else if ret, ok = responseHeaders[HEADER_SSEKMS_ENCRYPT_KEY_OBS]; ok { + sseKmsHeader.Key = ret[0] + } + sseHeader = sseKmsHeader + } + return +} + +func parseCorsHeader(output BaseModel) (AllowOrigin, AllowHeader, AllowMethod, ExposeHeader string, MaxAgeSeconds int) { + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { + AllowOrigin = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { + AllowHeader = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { + MaxAgeSeconds = StringToInt(ret[0], 0) + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { + AllowMethod = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { + ExposeHeader = ret[0] + } + return +} + +func parseUnCommonHeader(output *GetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRATION]; ok { + output.Expiration = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_RESTORE]; ok { + output.Restore = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_OBJECT_TYPE]; ok { + output.ObjectType = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_NEXT_APPEND_POSITION]; ok { + output.NextAppendPosition = ret[0] + } +} + +// ParseGetObjectMetadataOutput sets GetObjectMetadataOutput field values with response headers +func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { + output.AllowOrigin, output.AllowHeader, output.AllowMethod, output.ExposeHeader, output.MaxAgeSeconds = parseCorsHeader(output.BaseModel) + parseUnCommonHeader(output) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } + + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_LASTMODIFIED]; ok { + ret, err := time.Parse(time.RFC1123, ret[0]) + if err == nil { + output.LastModified = ret + } + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LENGTH]; ok { + output.ContentLength = StringToInt64(ret[0], 0) + } + + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } + +} + +// ParseCopyObjectOutput sets CopyObjectOutput field values with response headers +func ParseCopyObjectOutput(output *CopyObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_COPY_SOURCE_VERSION_ID]; ok { + output.CopySourceVersionId = ret[0] + } +} + +// ParsePutObjectOutput sets PutObjectOutput field values with response headers +func ParsePutObjectOutput(output *PutObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} + +// ParseInitiateMultipartUploadOutput sets InitiateMultipartUploadOutput field values with response headers +func ParseInitiateMultipartUploadOutput(output *InitiateMultipartUploadOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) +} + +// ParseUploadPartOutput sets UploadPartOutput field values with response headers +func ParseUploadPartOutput(output *UploadPartOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} + +// ParseCompleteMultipartUploadOutput sets CompleteMultipartUploadOutput field values with response headers +func ParseCompleteMultipartUploadOutput(output *CompleteMultipartUploadOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } +} + +// ParseCopyPartOutput sets CopyPartOutput field values with response headers +func ParseCopyPartOutput(output *CopyPartOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) +} + +// ParseGetBucketMetadataOutput sets GetBucketMetadataOutput field values with response headers +func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { + output.AllowOrigin, output.AllowHeader, output.AllowMethod, output.ExposeHeader, output.MaxAgeSeconds = parseCorsHeader(output.BaseModel) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_VERSION_OBS]; ok { + output.Version = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = ret[0] + } else if ret, ok := output.ResponseHeaders[HEADER_BUCKET_LOCATION_OBS]; ok { + output.Location = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EPID_HEADERS]; ok { + output.Epid = ret[0] + } + if ret, ok := output.ResponseHeaders[headerFSFileInterface]; ok { + output.FSStatus = parseStringToFSStatusType(ret[0]) + } else { + output.FSStatus = FSStatusDisabled + } +} + +func parseContentHeader(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } +} + +// ParseSetObjectMetadataOutput sets SetObjectMetadataOutput field values with response headers +func ParseSetObjectMetadataOutput(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_METADATA_DIRECTIVE]; ok { + output.MetadataDirective = MetadataDirectiveType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + parseContentHeader(output) + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } +} + +// ParseDeleteObjectOutput sets DeleteObjectOutput field values with response headers +func ParseDeleteObjectOutput(output *DeleteObjectOutput) { + if versionID, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionID[0] + } + + if deleteMarker, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok { + output.DeleteMarker = deleteMarker[0] == "true" + } +} + +// ParseGetObjectOutput sets GetObjectOutput field values with response headers +func ParseGetObjectOutput(output *GetObjectOutput) { + ParseGetObjectMetadataOutput(&output.GetObjectMetadataOutput) + if ret, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok { + output.DeleteMarker = ret[0] == "true" + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } +} + +// ConvertRequestToIoReaderV2 converts req to XML data +func ConvertRequestToIoReaderV2(req interface{}) (io.Reader, string, error) { + data, err := TransToXml(req) + if err == nil { + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "Do http request with data: %s", string(data)) + } + return bytes.NewReader(data), Base64Md5(data), nil + } + return nil, "", err +} + +// ConvertRequestToIoReader converts req to XML data +func ConvertRequestToIoReader(req interface{}) (io.Reader, error) { + body, err := TransToXml(req) + if err == nil { + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "Do http request with data: %s", string(body)) + } + return bytes.NewReader(body), nil + } + return nil, err +} + +// ParseResponseToBaseModel gets response from OBS +func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool, isObs bool) (err error) { + readCloser, ok := baseModel.(IReadCloser) + if !ok { + defer func() { + errMsg := resp.Body.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close response body") + } + }() + body, err := ioutil.ReadAll(resp.Body) + if err == nil && len(body) > 0 { + if xmlResult { + err = ParseXml(body, baseModel) + } else { + s := reflect.TypeOf(baseModel).Elem() + if reflect.TypeOf(baseModel).Elem().Name() == "GetBucketPolicyOutput" { + for i := 0; i < s.NumField(); i++ { + if s.Field(i).Tag == "json:\"body\"" { + reflect.ValueOf(baseModel).Elem().FieldByName(s.Field(i).Name).SetString(string(body)) + break + } + } + } else { + err = parseJSON(body, baseModel) + } + } + if err != nil { + doLog(LEVEL_ERROR, "Unmarshal error: %v", err) + } + } + } else { + readCloser.setReadCloser(resp.Body) + } + + baseModel.setStatusCode(resp.StatusCode) + responseHeaders := cleanHeaderPrefix(resp.Header) + baseModel.setResponseHeaders(responseHeaders) + if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok { + baseModel.setRequestID(values[0]) + } + return +} + +// ParseResponseToObsError gets obsError from OBS +func ParseResponseToObsError(resp *http.Response, isObs bool) error { + isJson := false + if contentType, ok := resp.Header[HEADER_CONTENT_TYPE_CAML]; ok { + jsonType, _ := mimeTypes["json"] + isJson = contentType[0] == jsonType + } + obsError := ObsError{} + respError := ParseResponseToBaseModel(resp, &obsError, !isJson, isObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + obsError.Status = resp.Status + return obsError +} + +// convertFetchPolicyToJSON converts SetBucketFetchPolicyInput into json format +func convertFetchPolicyToJSON(input SetBucketFetchPolicyInput) (data string, err error) { + fetch := map[string]SetBucketFetchPolicyInput{"fetch": input} + json, err := json.Marshal(fetch) + if err != nil { + return "", err + } + data = string(json) + return +} + +// convertFetchJobToJSON converts SetBucketFetchJobInput into json format +func convertFetchJobToJSON(input SetBucketFetchJobInput) (data string, err error) { + objectHeaders := make(map[string]string) + for key, value := range input.ObjectHeaders { + if value != "" { + _key := strings.ToLower(key) + if !strings.HasPrefix(key, HEADER_PREFIX_OBS) { + _key = HEADER_PREFIX_META_OBS + _key + } + objectHeaders[_key] = value + } + } + input.ObjectHeaders = objectHeaders + json, err := json.Marshal(input) + if err != nil { + return "", err + } + data = string(json) + return +} + +func parseStringToFSStatusType(value string) (ret FSStatusType) { + switch value { + case "Enabled": + ret = FSStatusEnabled + case "Disabled": + ret = FSStatusDisabled + default: + ret = "" + } + return +} diff --git a/cve-vulner-manager/obs/error.go b/cve-vulner-manager/obs/error.go new file mode 100644 index 0000000000000000000000000000000000000000..eb8ab280992841cf53d44ced5aefda20d46fcf2a --- /dev/null +++ b/cve-vulner-manager/obs/error.go @@ -0,0 +1,34 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "encoding/xml" + "fmt" +) + +// ObsError defines error response from OBS +type ObsError struct { + BaseModel + Status string + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code" json:"code"` + Message string `xml:"Message" json:"message"` + Resource string `xml:"Resource"` + HostId string `xml:"HostId"` +} + +func (err ObsError) Error() string { + return fmt.Sprintf("obs: service returned error: Status=%s, Code=%s, Message=%s, RequestId=%s", + err.Status, err.Code, err.Message, err.RequestId) +} diff --git a/cve-vulner-manager/obs/extension.go b/cve-vulner-manager/obs/extension.go new file mode 100644 index 0000000000000000000000000000000000000000..aa4ab02a385451650ff0a410cd0b5115e506f698 --- /dev/null +++ b/cve-vulner-manager/obs/extension.go @@ -0,0 +1,36 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "strings" +) + +type extensionOptions interface{} +type extensionHeaders func(headers map[string][]string, isObs bool) error + +func setHeaderPrefix(key string, value string) extensionHeaders { + return func(headers map[string][]string, isObs bool) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("set header %s with empty value", key) + } + setHeaders(headers, key, []string{value}, isObs) + return nil + } +} + +// WithReqPaymentHeader sets header for requester-pays +func WithReqPaymentHeader(requester PayerType) extensionHeaders { + return setHeaderPrefix(REQUEST_PAYER, string(requester)) +} diff --git a/cve-vulner-manager/obs/http.go b/cve-vulner-manager/obs/http.go new file mode 100644 index 0000000000000000000000000000000000000000..9e501de9ee6e55a5d487dd5e0799d05436bc05c9 --- /dev/null +++ b/cve-vulner-manager/obs/http.go @@ -0,0 +1,576 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +func prepareHeaders(headers map[string][]string, meta bool, isObs bool) map[string][]string { + _headers := make(map[string][]string, len(headers)) + if headers != nil { + for key, value := range headers { + key = strings.TrimSpace(key) + if key == "" { + continue + } + _key := strings.ToLower(key) + if _, ok := allowedRequestHTTPHeaderMetadataNames[_key]; !ok && !strings.HasPrefix(key, HEADER_PREFIX) && !strings.HasPrefix(key, HEADER_PREFIX_OBS) { + if !meta { + continue + } + if !isObs { + _key = HEADER_PREFIX_META + _key + } else { + _key = HEADER_PREFIX_META_OBS + _key + } + } else { + _key = key + } + _headers[_key] = value + } + } + return _headers +} + +func (obsClient ObsClient) doActionWithoutBucket(action, method string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + return obsClient.doAction(action, method, "", "", input, output, true, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketV2(action, method, bucketName string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + return obsClient.doAction(action, method, bucketName, "", input, output, false, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucket(action, method, bucketName string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + return obsClient.doAction(action, method, bucketName, "", input, output, true, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketAndKeyV2(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + if strings.TrimSpace(objectKey) == "" { + return errors.New("Key is empty") + } + return obsClient.doAction(action, method, bucketName, objectKey, input, output, false, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketAndKeyUnRepeatable(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, false, extensions) +} + +func (obsClient ObsClient) _doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, repeatable bool, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + if strings.TrimSpace(objectKey) == "" { + return errors.New("Key is empty") + } + return obsClient.doAction(action, method, bucketName, objectKey, input, output, true, repeatable, extensions) +} + +func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, xmlResult bool, repeatable bool, extensions []extensionOptions) error { + + var resp *http.Response + var respError error + doLog(LEVEL_INFO, "Enter method %s...", action) + start := GetCurrentTimestamp() + + params, headers, data, err := input.trans(obsClient.conf.signature == SignatureObs) + if err != nil { + return err + } + + if params == nil { + params = make(map[string]string) + } + + if headers == nil { + headers = make(map[string][]string) + } + + for _, extension := range extensions { + if extensionHeader, ok := extension.(extensionHeaders); ok { + _err := extensionHeader(headers, obsClient.conf.signature == SignatureObs) + if _err != nil { + doLog(LEVEL_INFO, fmt.Sprintf("set header with error: %v", _err)) + } + } else { + doLog(LEVEL_INFO, "Unsupported extensionOptions") + } + } + + switch method { + case HTTP_GET: + resp, respError = obsClient.doHTTPGet(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_POST: + resp, respError = obsClient.doHTTPPost(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_PUT: + resp, respError = obsClient.doHTTPPut(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_DELETE: + resp, respError = obsClient.doHTTPDelete(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_HEAD: + resp, respError = obsClient.doHTTPHead(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_OPTIONS: + resp, respError = obsClient.doHTTPOptions(bucketName, objectKey, params, headers, data, repeatable) + default: + respError = errors.New("Unexpect http method error") + } + if respError == nil && output != nil { + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + } else { + doLog(LEVEL_WARN, "Do http request with error: %v", respError) + } + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } + + return respError +} + +func (obsClient ObsClient) doHTTPGet(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_GET, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPHead(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_HEAD, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPOptions(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_OPTIONS, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPDelete(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_DELETE, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPPut(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_PUT, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPPost(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_POST, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func prepareAgentHeader(clientUserAgent string) string { + userAgent := USER_AGENT + if clientUserAgent != "" { + userAgent = clientUserAgent + } + return userAgent +} + +func (obsClient ObsClient) doHTTPWithSignedURL(action, method string, signedURL string, actualSignedRequestHeaders http.Header, data io.Reader, output IBaseModel, xmlResult bool) (respError error) { + req, err := http.NewRequest(method, signedURL, data) + if err != nil { + return err + } + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } + var resp *http.Response + + var isSecurityToken bool + var securityToken string + var query []string + parmas := strings.Split(signedURL, "?") + if len(parmas) > 1 { + query = strings.Split(parmas[1], "&") + for _, value := range query { + if strings.HasPrefix(value, HEADER_STS_TOKEN_AMZ+"=") || strings.HasPrefix(value, HEADER_STS_TOKEN_OBS+"=") { + if value[len(HEADER_STS_TOKEN_AMZ)+1:] != "" { + securityToken = value[len(HEADER_STS_TOKEN_AMZ)+1:] + isSecurityToken = true + } + } + } + } + logSignedURL := signedURL + if isSecurityToken { + logSignedURL = strings.Replace(logSignedURL, securityToken, "******", -1) + } + doLog(LEVEL_INFO, "Do %s with signedUrl %s...", action, logSignedURL) + + req.Header = actualSignedRequestHeaders + if value, ok := req.Header[HEADER_HOST_CAMEL]; ok { + req.Host = value[0] + delete(req.Header, HEADER_HOST_CAMEL) + } else if value, ok := req.Header[HEADER_HOST]; ok { + req.Host = value[0] + delete(req.Header, HEADER_HOST) + } + + if value, ok := req.Header[HEADER_CONTENT_LENGTH_CAMEL]; ok { + req.ContentLength = StringToInt64(value[0], -1) + delete(req.Header, HEADER_CONTENT_LENGTH_CAMEL) + } else if value, ok := req.Header[HEADER_CONTENT_LENGTH]; ok { + req.ContentLength = StringToInt64(value[0], -1) + delete(req.Header, HEADER_CONTENT_LENGTH) + } + + userAgent := prepareAgentHeader(obsClient.conf.userAgent) + req.Header[HEADER_USER_AGENT_CAMEL] = []string{userAgent} + start := GetCurrentTimestamp() + resp, err = obsClient.httpClient.Do(req) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } + + var msg interface{} + if err != nil { + respError = err + resp = nil + } else { + doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) + if resp.StatusCode >= 300 { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + msg = resp.Status + resp = nil + } else { + if output != nil { + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) + } + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + } + } + + if msg != nil { + doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) + } + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } + + return +} + +func (obsClient ObsClient) doHTTP(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (resp *http.Response, respError error) { + + bucketName = strings.TrimSpace(bucketName) + + method = strings.ToUpper(method) + + var redirectURL string + var requestURL string + maxRetryCount := obsClient.conf.maxRetryCount + maxRedirectCount := obsClient.conf.maxRedirectCount + + var _data io.Reader + if data != nil { + if dataStr, ok := data.(string); ok { + doLog(LEVEL_DEBUG, "Do http request with string: %s", dataStr) + headers["Content-Length"] = []string{IntToString(len(dataStr))} + _data = strings.NewReader(dataStr) + } else if dataByte, ok := data.([]byte); ok { + doLog(LEVEL_DEBUG, "Do http request with byte array") + headers["Content-Length"] = []string{IntToString(len(dataByte))} + _data = bytes.NewReader(dataByte) + } else if dataReader, ok := data.(io.Reader); ok { + _data = dataReader + } else { + doLog(LEVEL_WARN, "Data is not a valid io.Reader") + return nil, errors.New("Data is not a valid io.Reader") + } + } + + var lastRequest *http.Request + redirectFlag := false + for i, redirectCount := 0, 0; i <= maxRetryCount; i++ { + if redirectURL != "" { + if !redirectFlag { + parsedRedirectURL, err := url.Parse(redirectURL) + if err != nil { + return nil, err + } + requestURL, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectURL.Host) + if err != nil { + return nil, err + } + if parsedRequestURL, err := url.Parse(requestURL); err != nil { + return nil, err + } else if parsedRequestURL.RawQuery != "" && parsedRedirectURL.RawQuery == "" { + redirectURL += "?" + parsedRequestURL.RawQuery + } + } + requestURL = redirectURL + } else { + var err error + requestURL, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, "") + if err != nil { + return nil, err + } + } + + req, err := http.NewRequest(method, requestURL, _data) + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } + if err != nil { + return nil, err + } + doLog(LEVEL_DEBUG, "Do request with url [%s] and method [%s]", requestURL, method) + + if isDebugLogEnabled() { + auth := headers[HEADER_AUTH_CAMEL] + delete(headers, HEADER_AUTH_CAMEL) + + var isSecurityToken bool + var securityToken []string + if securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_AMZ]; isSecurityToken { + headers[HEADER_STS_TOKEN_AMZ] = []string{"******"} + } else if securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_OBS]; isSecurityToken { + headers[HEADER_STS_TOKEN_OBS] = []string{"******"} + } + doLog(LEVEL_DEBUG, "Request headers: %v", headers) + headers[HEADER_AUTH_CAMEL] = auth + if isSecurityToken { + if obsClient.conf.signature == SignatureObs { + headers[HEADER_STS_TOKEN_OBS] = securityToken + } else { + headers[HEADER_STS_TOKEN_AMZ] = securityToken + } + } + } + + for key, value := range headers { + if key == HEADER_HOST_CAMEL { + req.Host = value[0] + delete(headers, key) + } else if key == HEADER_CONTENT_LENGTH_CAMEL { + req.ContentLength = StringToInt64(value[0], -1) + delete(headers, key) + } else { + req.Header[key] = value + } + } + + lastRequest = req + + userAgent := prepareAgentHeader(obsClient.conf.userAgent) + req.Header[HEADER_USER_AGENT_CAMEL] = []string{userAgent} + + if lastRequest != nil { + req.Host = lastRequest.Host + req.ContentLength = lastRequest.ContentLength + } + + start := GetCurrentTimestamp() + resp, err = obsClient.httpClient.Do(req) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } + + var msg interface{} + if err != nil { + msg = err + respError = err + resp = nil + if !repeatable { + break + } + } else { + doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) + if resp.StatusCode < 300 { + break + } else if !repeatable || (resp.StatusCode >= 400 && resp.StatusCode < 500) || resp.StatusCode == 304 { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + break + } else if resp.StatusCode >= 300 && resp.StatusCode < 400 { + if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" && redirectCount < maxRedirectCount { + redirectURL = location + doLog(LEVEL_WARN, "Redirect request to %s", redirectURL) + msg = resp.Status + maxRetryCount++ + redirectCount++ + if resp.StatusCode == 302 && method == HTTP_GET { + redirectFlag = true + } else { + redirectFlag = false + } + } else { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + break + } + } else { + msg = resp.Status + } + } + if i != maxRetryCount { + if resp != nil { + _err := resp.Body.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close resp body") + } + resp = nil + } + if _, ok := headers[HEADER_AUTH_CAMEL]; ok { + delete(headers, HEADER_AUTH_CAMEL) + } + doLog(LEVEL_WARN, "Failed to send request with reason:%v, will try again", msg) + if r, ok := _data.(*strings.Reader); ok { + _, err := r.Seek(0, 0) + if err != nil { + return nil, err + } + } else if r, ok := _data.(*bytes.Reader); ok { + _, err := r.Seek(0, 0) + if err != nil { + return nil, err + } + } else if r, ok := _data.(*fileReaderWrapper); ok { + fd, err := os.Open(r.filePath) + if err != nil { + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close with reason: %v", errMsg) + } + }() + fileReaderWrapper := &fileReaderWrapper{filePath: r.filePath} + fileReaderWrapper.mark = r.mark + fileReaderWrapper.reader = fd + fileReaderWrapper.totalCount = r.totalCount + _data = fileReaderWrapper + _, err = fd.Seek(r.mark, 0) + if err != nil { + return nil, err + } + } else if r, ok := _data.(*readerWrapper); ok { + _, err := r.seek(0, 0) + if err != nil { + return nil, err + } + } + time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second))) + } else { + doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) + if resp != nil { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + } + } + } + return +} + +type connDelegate struct { + conn net.Conn + socketTimeout time.Duration + finalTimeout time.Duration +} + +func getConnDelegate(conn net.Conn, socketTimeout int, finalTimeout int) *connDelegate { + return &connDelegate{ + conn: conn, + socketTimeout: time.Second * time.Duration(socketTimeout), + finalTimeout: time.Second * time.Duration(finalTimeout), + } +} + +func (delegate *connDelegate) Read(b []byte) (n int, err error) { + setReadDeadlineErr := delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + + n, err = delegate.conn.Read(b) + setReadDeadlineErr = delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + return n, err +} + +func (delegate *connDelegate) Write(b []byte) (n int, err error) { + setWriteDeadlineErr := delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + + n, err = delegate.conn.Write(b) + finalTimeout := time.Now().Add(delegate.finalTimeout) + setWriteDeadlineErr = delegate.SetWriteDeadline(finalTimeout) + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + setReadDeadlineErr := delegate.SetReadDeadline(finalTimeout) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + return n, err +} + +func (delegate *connDelegate) Close() error { + return delegate.conn.Close() +} + +func (delegate *connDelegate) LocalAddr() net.Addr { + return delegate.conn.LocalAddr() +} + +func (delegate *connDelegate) RemoteAddr() net.Addr { + return delegate.conn.RemoteAddr() +} + +func (delegate *connDelegate) SetDeadline(t time.Time) error { + return delegate.conn.SetDeadline(t) +} + +func (delegate *connDelegate) SetReadDeadline(t time.Time) error { + return delegate.conn.SetReadDeadline(t) +} + +func (delegate *connDelegate) SetWriteDeadline(t time.Time) error { + return delegate.conn.SetWriteDeadline(t) +} diff --git a/cve-vulner-manager/obs/log.go b/cve-vulner-manager/obs/log.go new file mode 100644 index 0000000000000000000000000000000000000000..0750c80d5399795b6dd3f009960d33812ca0a6c6 --- /dev/null +++ b/cve-vulner-manager/obs/log.go @@ -0,0 +1,316 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + "sync" +) + +// Level defines the level of the log +type Level int + +const ( + LEVEL_OFF Level = 500 + LEVEL_ERROR Level = 400 + LEVEL_WARN Level = 300 + LEVEL_INFO Level = 200 + LEVEL_DEBUG Level = 100 +) + +var logLevelMap = map[Level]string{ + LEVEL_OFF: "[OFF]: ", + LEVEL_ERROR: "[ERROR]: ", + LEVEL_WARN: "[WARN]: ", + LEVEL_INFO: "[INFO]: ", + LEVEL_DEBUG: "[DEBUG]: ", +} + +type logConfType struct { + level Level + logToConsole bool + logFullPath string + maxLogSize int64 + backups int +} + +func getDefaultLogConf() logConfType { + return logConfType{ + level: LEVEL_WARN, + logToConsole: false, + logFullPath: "", + maxLogSize: 1024 * 1024 * 30, //30MB + backups: 10, + } +} + +var logConf logConfType + +type loggerWrapper struct { + fullPath string + fd *os.File + ch chan string + wg sync.WaitGroup + queue []string + logger *log.Logger + index int + cacheCount int + closed bool +} + +func (lw *loggerWrapper) doInit() { + lw.queue = make([]string, 0, lw.cacheCount) + lw.logger = log.New(lw.fd, "", 0) + lw.ch = make(chan string, lw.cacheCount) + lw.wg.Add(1) + go lw.doWrite() +} + +func (lw *loggerWrapper) rotate() { + stat, err := lw.fd.Stat() + if err != nil { + _err := lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + panic(err) + } + if stat.Size() >= logConf.maxLogSize { + _err := lw.fd.Sync() + if _err != nil { + panic(err) + } + _err = lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + if lw.index > logConf.backups { + lw.index = 1 + } + _err = os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) + if _err != nil { + panic(err) + } + lw.index++ + + fd, err := os.OpenFile(lw.fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + panic(err) + } + lw.fd = fd + lw.logger.SetOutput(lw.fd) + } +} + +func (lw *loggerWrapper) doFlush() { + lw.rotate() + for _, m := range lw.queue { + lw.logger.Println(m) + } + err := lw.fd.Sync() + if err != nil { + panic(err) + } +} + +func (lw *loggerWrapper) doClose() { + lw.closed = true + close(lw.ch) + lw.wg.Wait() +} + +func (lw *loggerWrapper) doWrite() { + defer lw.wg.Done() + for { + msg, ok := <-lw.ch + if !ok { + lw.doFlush() + _err := lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + break + } + if len(lw.queue) >= lw.cacheCount { + lw.doFlush() + lw.queue = make([]string, 0, lw.cacheCount) + } + lw.queue = append(lw.queue, msg) + } + +} + +func (lw *loggerWrapper) Printf(format string, v ...interface{}) { + if !lw.closed { + msg := fmt.Sprintf(format, v...) + lw.ch <- msg + } +} + +var consoleLogger *log.Logger +var fileLogger *loggerWrapper +var lock = new(sync.RWMutex) + +func isDebugLogEnabled() bool { + return logConf.level <= LEVEL_DEBUG +} + +func isErrorLogEnabled() bool { + return logConf.level <= LEVEL_ERROR +} + +func isWarnLogEnabled() bool { + return logConf.level <= LEVEL_WARN +} + +func isInfoLogEnabled() bool { + return logConf.level <= LEVEL_INFO +} + +func reset() { + if fileLogger != nil { + fileLogger.doClose() + fileLogger = nil + } + consoleLogger = nil + logConf = getDefaultLogConf() +} + +// InitLog enable logging function with default cacheCnt +func InitLog(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool) error { + return InitLogWithCacheCnt(logFullPath, maxLogSize, backups, level, logToConsole, 50) +} + +// InitLogWithCacheCnt enable logging function +func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool, cacheCnt int) error { + lock.Lock() + defer lock.Unlock() + if cacheCnt <= 0 { + cacheCnt = 50 + } + reset() + if fullPath := strings.TrimSpace(logFullPath); fullPath != "" { + _fullPath, err := filepath.Abs(fullPath) + if err != nil { + return err + } + + if !strings.HasSuffix(_fullPath, ".log") { + _fullPath += ".log" + } + + stat, err := os.Stat(_fullPath) + if err == nil && stat.IsDir() { + return fmt.Errorf("logFullPath:[%s] is a directory", _fullPath) + } else if err = os.MkdirAll(filepath.Dir(_fullPath), os.ModePerm); err != nil { + return err + } + + fd, err := os.OpenFile(_fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return err + } + + if stat == nil { + stat, err = os.Stat(_fullPath) + if err != nil { + _err := fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + return err + } + } + + prefix := stat.Name() + "." + index := 1 + var timeIndex int64 = 0 + walkFunc := func(path string, info os.FileInfo, err error) error { + if err == nil { + if name := info.Name(); strings.HasPrefix(name, prefix) { + if i := StringToInt(name[len(prefix):], 0); i >= index && info.ModTime().Unix() >= timeIndex { + timeIndex = info.ModTime().Unix() + index = i + 1 + } + } + } + return err + } + + if err = filepath.Walk(filepath.Dir(_fullPath), walkFunc); err != nil { + _err := fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + return err + } + + fileLogger = &loggerWrapper{fullPath: _fullPath, fd: fd, index: index, cacheCount: cacheCnt, closed: false} + fileLogger.doInit() + } + if maxLogSize > 0 { + logConf.maxLogSize = maxLogSize + } + if backups > 0 { + logConf.backups = backups + } + logConf.level = level + if logToConsole { + consoleLogger = log.New(os.Stdout, "", log.LstdFlags) + } + return nil +} + +// CloseLog disable logging and synchronize cache data to log files +func CloseLog() { + if logEnabled() { + lock.Lock() + defer lock.Unlock() + reset() + } +} + +func logEnabled() bool { + return consoleLogger != nil || fileLogger != nil +} + +// DoLog writes log messages to the logger +func DoLog(level Level, format string, v ...interface{}) { + doLog(level, format, v...) +} + +func doLog(level Level, format string, v ...interface{}) { + if logEnabled() && logConf.level <= level { + msg := fmt.Sprintf(format, v...) + if _, file, line, ok := runtime.Caller(1); ok { + index := strings.LastIndex(file, "/") + if index >= 0 { + file = file[index+1:] + } + msg = fmt.Sprintf("%s:%d|%s", file, line, msg) + } + prefix := logLevelMap[level] + if consoleLogger != nil { + consoleLogger.Printf("%s%s", prefix, msg) + } + if fileLogger != nil { + nowDate := FormatUtcNow("2006-01-02T15:04:05Z") + fileLogger.Printf("%s %s%s", nowDate, prefix, msg) + } + } +} diff --git a/cve-vulner-manager/obs/model.go b/cve-vulner-manager/obs/model.go new file mode 100644 index 0000000000000000000000000000000000000000..f04f62717429818c4e9f8b29ba2e38dd847bc7bb --- /dev/null +++ b/cve-vulner-manager/obs/model.go @@ -0,0 +1,1238 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "encoding/xml" + "io" + "net/http" + "time" +) + +// BaseModel defines base model response from OBS +type BaseModel struct { + StatusCode int `xml:"-"` + RequestId string `xml:"RequestId" json:"request_id"` + ResponseHeaders map[string][]string `xml:"-"` +} + +// Bucket defines bucket properties +type Bucket struct { + XMLName xml.Name `xml:"Bucket"` + Name string `xml:"Name"` + CreationDate time.Time `xml:"CreationDate"` + Location string `xml:"Location"` + BucketType string `xml:"BucketType,omitempty"` +} + +// Owner defines owner properties +type Owner struct { + XMLName xml.Name `xml:"Owner"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName,omitempty"` +} + +// Initiator defines initiator properties +type Initiator struct { + XMLName xml.Name `xml:"Initiator"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName,omitempty"` +} + +// ListBucketsInput is the input parameter of ListBuckets function +type ListBucketsInput struct { + QueryLocation bool +} + +// ListBucketsOutput is the result of ListBuckets function +type ListBucketsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner Owner `xml:"Owner"` + Buckets []Bucket `xml:"Buckets>Bucket"` +} + +type bucketLocationObs struct { + XMLName xml.Name `xml:"Location"` + Location string `xml:",chardata"` +} + +// BucketLocation defines bucket location configuration +type BucketLocation struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + Location string `xml:"LocationConstraint,omitempty"` +} + +// CreateBucketInput is the input parameter of CreateBucket function +type CreateBucketInput struct { + BucketLocation + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + StorageClass StorageClassType `xml:"-"` + GrantReadId string `xml:"-"` + GrantWriteId string `xml:"-"` + GrantReadAcpId string `xml:"-"` + GrantWriteAcpId string `xml:"-"` + GrantFullControlId string `xml:"-"` + GrantReadDeliveredId string `xml:"-"` + GrantFullControlDeliveredId string `xml:"-"` + Epid string `xml:"-"` + AvailableZone string `xml:"-"` + IsFSFileInterface bool `xml:"-"` +} + +// BucketStoragePolicy defines the bucket storage class +type BucketStoragePolicy struct { + XMLName xml.Name `xml:"StoragePolicy"` + StorageClass StorageClassType `xml:"DefaultStorageClass"` +} + +// SetBucketStoragePolicyInput is the input parameter of SetBucketStoragePolicy function +type SetBucketStoragePolicyInput struct { + Bucket string `xml:"-"` + BucketStoragePolicy +} + +type getBucketStoragePolicyOutputS3 struct { + BaseModel + BucketStoragePolicy +} + +// GetBucketStoragePolicyOutput is the result of GetBucketStoragePolicy function +type GetBucketStoragePolicyOutput struct { + BaseModel + StorageClass string +} + +type bucketStoragePolicyObs struct { + XMLName xml.Name `xml:"StorageClass"` + StorageClass string `xml:",chardata"` +} +type getBucketStoragePolicyOutputObs struct { + BaseModel + bucketStoragePolicyObs +} + +// ListObjsInput defines parameters for listing objects +type ListObjsInput struct { + Prefix string + MaxKeys int + Delimiter string + Origin string + RequestHeader string +} + +// ListObjectsInput is the input parameter of ListObjects function +type ListObjectsInput struct { + ListObjsInput + Bucket string + Marker string +} + +// Content defines the object content properties +type Content struct { + XMLName xml.Name `xml:"Contents"` + Owner Owner `xml:"Owner"` + ETag string `xml:"ETag"` + Key string `xml:"Key"` + LastModified time.Time `xml:"LastModified"` + Size int64 `xml:"Size"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// ListObjectsOutput is the result of ListObjects function +type ListObjectsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListBucketResult"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + Marker string `xml:"Marker"` + NextMarker string `xml:"NextMarker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Contents []Content `xml:"Contents"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + Location string `xml:"-"` +} + +// ListVersionsInput is the input parameter of ListVersions function +type ListVersionsInput struct { + ListObjsInput + Bucket string + KeyMarker string + VersionIdMarker string +} + +// Version defines the properties of versioning objects +type Version struct { + DeleteMarker + XMLName xml.Name `xml:"Version"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` +} + +// DeleteMarker defines the properties of versioning delete markers +type DeleteMarker struct { + XMLName xml.Name `xml:"DeleteMarker"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified time.Time `xml:"LastModified"` + Owner Owner `xml:"Owner"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// ListVersionsOutput is the result of ListVersions function +type ListVersionsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListVersionsResult"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + VersionIdMarker string `xml:"VersionIdMarker"` + NextVersionIdMarker string `xml:"NextVersionIdMarker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Versions []Version `xml:"Version"` + DeleteMarkers []DeleteMarker `xml:"DeleteMarker"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + Location string `xml:"-"` +} + +// ListMultipartUploadsInput is the input parameter of ListMultipartUploads function +type ListMultipartUploadsInput struct { + Bucket string + Prefix string + MaxUploads int + Delimiter string + KeyMarker string + UploadIdMarker string +} + +// Upload defines multipart upload properties +type Upload struct { + XMLName xml.Name `xml:"Upload"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + Initiated time.Time `xml:"Initiated"` + StorageClass StorageClassType `xml:"StorageClass"` + Owner Owner `xml:"Owner"` + Initiator Initiator `xml:"Initiator"` +} + +// ListMultipartUploadsOutput is the result of ListMultipartUploads function +type ListMultipartUploadsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListMultipartUploadsResult"` + Bucket string `xml:"Bucket"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + UploadIdMarker string `xml:"UploadIdMarker"` + NextUploadIdMarker string `xml:"NextUploadIdMarker"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + MaxUploads int `xml:"MaxUploads"` + Prefix string `xml:"Prefix"` + Uploads []Upload `xml:"Upload"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` +} + +// BucketQuota defines bucket quota configuration +type BucketQuota struct { + XMLName xml.Name `xml:"Quota"` + Quota int64 `xml:"StorageQuota"` +} + +// SetBucketQuotaInput is the input parameter of SetBucketQuota function +type SetBucketQuotaInput struct { + Bucket string `xml:"-"` + BucketQuota +} + +// GetBucketQuotaOutput is the result of GetBucketQuota function +type GetBucketQuotaOutput struct { + BaseModel + BucketQuota +} + +// GetBucketStorageInfoOutput is the result of GetBucketStorageInfo function +type GetBucketStorageInfoOutput struct { + BaseModel + XMLName xml.Name `xml:"GetBucketStorageInfoResult"` + Size int64 `xml:"Size"` + ObjectNumber int `xml:"ObjectNumber"` +} + +type getBucketLocationOutputS3 struct { + BaseModel + BucketLocation +} +type getBucketLocationOutputObs struct { + BaseModel + bucketLocationObs +} + +// GetBucketLocationOutput is the result of GetBucketLocation function +type GetBucketLocationOutput struct { + BaseModel + Location string `xml:"-"` +} + +// Grantee defines grantee properties +type Grantee struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + URI GroupUriType `xml:"URI,omitempty"` +} + +type granteeObs struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + Canned string `xml:"Canned,omitempty"` +} + +// Grant defines grant properties +type Grant struct { + XMLName xml.Name `xml:"Grant"` + Grantee Grantee `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} +type grantObs struct { + XMLName xml.Name `xml:"Grant"` + Grantee granteeObs `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} + +// AccessControlPolicy defines access control policy properties +type AccessControlPolicy struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []Grant `xml:"AccessControlList>Grant"` + Delivered string `xml:"Delivered,omitempty"` +} + +type accessControlPolicyObs struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []grantObs `xml:"AccessControlList>Grant"` +} + +// GetBucketAclOutput is the result of GetBucketAcl function +type GetBucketAclOutput struct { + BaseModel + AccessControlPolicy +} + +type getBucketACLOutputObs struct { + BaseModel + accessControlPolicyObs +} + +// SetBucketAclInput is the input parameter of SetBucketAcl function +type SetBucketAclInput struct { + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + AccessControlPolicy +} + +// SetBucketPolicyInput is the input parameter of SetBucketPolicy function +type SetBucketPolicyInput struct { + Bucket string + Policy string +} + +// GetBucketPolicyOutput is the result of GetBucketPolicy function +type GetBucketPolicyOutput struct { + BaseModel + Policy string `json:"body"` +} + +// CorsRule defines the CORS rules +type CorsRule struct { + XMLName xml.Name `xml:"CORSRule"` + ID string `xml:"ID,omitempty"` + AllowedOrigin []string `xml:"AllowedOrigin"` + AllowedMethod []string `xml:"AllowedMethod"` + AllowedHeader []string `xml:"AllowedHeader,omitempty"` + MaxAgeSeconds int `xml:"MaxAgeSeconds"` + ExposeHeader []string `xml:"ExposeHeader,omitempty"` +} + +// BucketCors defines the bucket CORS configuration +type BucketCors struct { + XMLName xml.Name `xml:"CORSConfiguration"` + CorsRules []CorsRule `xml:"CORSRule"` +} + +// SetBucketCorsInput is the input parameter of SetBucketCors function +type SetBucketCorsInput struct { + Bucket string `xml:"-"` + BucketCors +} + +// GetBucketCorsOutput is the result of GetBucketCors function +type GetBucketCorsOutput struct { + BaseModel + BucketCors +} + +// BucketVersioningConfiguration defines the versioning configuration +type BucketVersioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status VersioningStatusType `xml:"Status"` +} + +// SetBucketVersioningInput is the input parameter of SetBucketVersioning function +type SetBucketVersioningInput struct { + Bucket string `xml:"-"` + BucketVersioningConfiguration +} + +// GetBucketVersioningOutput is the result of GetBucketVersioning function +type GetBucketVersioningOutput struct { + BaseModel + BucketVersioningConfiguration +} + +// IndexDocument defines the default page configuration +type IndexDocument struct { + Suffix string `xml:"Suffix"` +} + +// ErrorDocument defines the error page configuration +type ErrorDocument struct { + Key string `xml:"Key,omitempty"` +} + +// Condition defines condition in RoutingRule +type Condition struct { + XMLName xml.Name `xml:"Condition"` + KeyPrefixEquals string `xml:"KeyPrefixEquals,omitempty"` + HttpErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals,omitempty"` +} + +// Redirect defines redirect in RoutingRule +type Redirect struct { + XMLName xml.Name `xml:"Redirect"` + Protocol ProtocolType `xml:"Protocol,omitempty"` + HostName string `xml:"HostName,omitempty"` + ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith,omitempty"` + ReplaceKeyWith string `xml:"ReplaceKeyWith,omitempty"` + HttpRedirectCode string `xml:"HttpRedirectCode,omitempty"` +} + +// RoutingRule defines routing rules +type RoutingRule struct { + XMLName xml.Name `xml:"RoutingRule"` + Condition Condition `xml:"Condition,omitempty"` + Redirect Redirect `xml:"Redirect"` +} + +// RedirectAllRequestsTo defines redirect in BucketWebsiteConfiguration +type RedirectAllRequestsTo struct { + XMLName xml.Name `xml:"RedirectAllRequestsTo"` + Protocol ProtocolType `xml:"Protocol,omitempty"` + HostName string `xml:"HostName"` +} + +// BucketWebsiteConfiguration defines the bucket website configuration +type BucketWebsiteConfiguration struct { + XMLName xml.Name `xml:"WebsiteConfiguration"` + RedirectAllRequestsTo RedirectAllRequestsTo `xml:"RedirectAllRequestsTo,omitempty"` + IndexDocument IndexDocument `xml:"IndexDocument,omitempty"` + ErrorDocument ErrorDocument `xml:"ErrorDocument,omitempty"` + RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` +} + +// SetBucketWebsiteConfigurationInput is the input parameter of SetBucketWebsiteConfiguration function +type SetBucketWebsiteConfigurationInput struct { + Bucket string `xml:"-"` + BucketWebsiteConfiguration +} + +// GetBucketWebsiteConfigurationOutput is the result of GetBucketWebsiteConfiguration function +type GetBucketWebsiteConfigurationOutput struct { + BaseModel + BucketWebsiteConfiguration +} + +// GetBucketMetadataInput is the input parameter of GetBucketMetadata function +type GetBucketMetadataInput struct { + Bucket string + Origin string + RequestHeader string +} + +// SetObjectMetadataInput is the input parameter of SetObjectMetadata function +type SetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + +//SetObjectMetadataOutput is the result of SetObjectMetadata function +type SetObjectMetadataOutput struct { + BaseModel + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + +// GetBucketMetadataOutput is the result of GetBucketMetadata function +type GetBucketMetadataOutput struct { + BaseModel + StorageClass StorageClassType + Location string + Version string + AllowOrigin string + AllowMethod string + AllowHeader string + MaxAgeSeconds int + ExposeHeader string + Epid string + FSStatus FSStatusType +} + +// BucketLoggingStatus defines the bucket logging configuration +type BucketLoggingStatus struct { + XMLName xml.Name `xml:"BucketLoggingStatus"` + Agency string `xml:"Agency,omitempty"` + TargetBucket string `xml:"LoggingEnabled>TargetBucket,omitempty"` + TargetPrefix string `xml:"LoggingEnabled>TargetPrefix,omitempty"` + TargetGrants []Grant `xml:"LoggingEnabled>TargetGrants>Grant,omitempty"` +} + +// SetBucketLoggingConfigurationInput is the input parameter of SetBucketLoggingConfiguration function +type SetBucketLoggingConfigurationInput struct { + Bucket string `xml:"-"` + BucketLoggingStatus +} + +// GetBucketLoggingConfigurationOutput is the result of GetBucketLoggingConfiguration function +type GetBucketLoggingConfigurationOutput struct { + BaseModel + BucketLoggingStatus +} + +// Transition defines transition property in LifecycleRule +type Transition struct { + XMLName xml.Name `xml:"Transition"` + Date time.Time `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// Expiration defines expiration property in LifecycleRule +type Expiration struct { + XMLName xml.Name `xml:"Expiration"` + Date time.Time `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` +} + +// NoncurrentVersionTransition defines noncurrentVersion transition property in LifecycleRule +type NoncurrentVersionTransition struct { + XMLName xml.Name `xml:"NoncurrentVersionTransition"` + NoncurrentDays int `xml:"NoncurrentDays"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// NoncurrentVersionExpiration defines noncurrentVersion expiration property in LifecycleRule +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration"` + NoncurrentDays int `xml:"NoncurrentDays"` +} + +// LifecycleRule defines lifecycle rule +type LifecycleRule struct { + ID string `xml:"ID,omitempty"` + Prefix string `xml:"Prefix"` + Status RuleStatusType `xml:"Status"` + Transitions []Transition `xml:"Transition,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty"` + NoncurrentVersionTransitions []NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` +} + +// BucketLifecyleConfiguration defines the bucket lifecycle configuration +type BucketLifecyleConfiguration struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + LifecycleRules []LifecycleRule `xml:"Rule"` +} + +// SetBucketLifecycleConfigurationInput is the input parameter of SetBucketLifecycleConfiguration function +type SetBucketLifecycleConfigurationInput struct { + Bucket string `xml:"-"` + BucketLifecyleConfiguration +} + +// GetBucketLifecycleConfigurationOutput is the result of GetBucketLifecycleConfiguration function +type GetBucketLifecycleConfigurationOutput struct { + BaseModel + BucketLifecyleConfiguration +} + +// Tag defines tag property in BucketTagging +type Tag struct { + XMLName xml.Name `xml:"Tag"` + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +// BucketTagging defines the bucket tag configuration +type BucketTagging struct { + XMLName xml.Name `xml:"Tagging"` + Tags []Tag `xml:"TagSet>Tag"` +} + +// SetBucketTaggingInput is the input parameter of SetBucketTagging function +type SetBucketTaggingInput struct { + Bucket string `xml:"-"` + BucketTagging +} + +// GetBucketTaggingOutput is the result of GetBucketTagging function +type GetBucketTaggingOutput struct { + BaseModel + BucketTagging +} + +// FilterRule defines filter rule in TopicConfiguration +type FilterRule struct { + XMLName xml.Name `xml:"FilterRule"` + Name string `xml:"Name,omitempty"` + Value string `xml:"Value,omitempty"` +} + +// TopicConfiguration defines the topic configuration +type TopicConfiguration struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []EventType `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>Object>FilterRule"` +} + +// BucketNotification defines the bucket notification configuration +type BucketNotification struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []TopicConfiguration `xml:"TopicConfiguration"` +} + +// SetBucketNotificationInput is the input parameter of SetBucketNotification function +type SetBucketNotificationInput struct { + Bucket string `xml:"-"` + BucketNotification +} + +type topicConfigurationS3 struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []string `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>S3Key>FilterRule"` +} + +type bucketNotificationS3 struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []topicConfigurationS3 `xml:"TopicConfiguration"` +} + +type getBucketNotificationOutputS3 struct { + BaseModel + bucketNotificationS3 +} + +// GetBucketNotificationOutput is the result of GetBucketNotification function +type GetBucketNotificationOutput struct { + BaseModel + BucketNotification +} + +// DeleteObjectInput is the input parameter of DeleteObject function +type DeleteObjectInput struct { + Bucket string + Key string + VersionId string +} + +// DeleteObjectOutput is the result of DeleteObject function +type DeleteObjectOutput struct { + BaseModel + VersionId string + DeleteMarker bool +} + +// ObjectToDelete defines the object property in DeleteObjectsInput +type ObjectToDelete struct { + XMLName xml.Name `xml:"Object"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` +} + +// DeleteObjectsInput is the input parameter of DeleteObjects function +type DeleteObjectsInput struct { + Bucket string `xml:"-"` + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet,omitempty"` + Objects []ObjectToDelete `xml:"Object"` +} + +// Deleted defines the deleted property in DeleteObjectsOutput +type Deleted struct { + XMLName xml.Name `xml:"Deleted"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + DeleteMarker bool `xml:"DeleteMarker"` + DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId"` +} + +// Error defines the error property in DeleteObjectsOutput +type Error struct { + XMLName xml.Name `xml:"Error"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +// DeleteObjectsOutput is the result of DeleteObjects function +type DeleteObjectsOutput struct { + BaseModel + XMLName xml.Name `xml:"DeleteResult"` + Deleteds []Deleted `xml:"Deleted"` + Errors []Error `xml:"Error"` +} + +// SetObjectAclInput is the input parameter of SetObjectAcl function +type SetObjectAclInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + VersionId string `xml:"-"` + ACL AclType `xml:"-"` + AccessControlPolicy +} + +// GetObjectAclInput is the input parameter of GetObjectAcl function +type GetObjectAclInput struct { + Bucket string + Key string + VersionId string +} + +// GetObjectAclOutput is the result of GetObjectAcl function +type GetObjectAclOutput struct { + BaseModel + VersionId string + AccessControlPolicy +} + +// RestoreObjectInput is the input parameter of RestoreObject function +type RestoreObjectInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + VersionId string `xml:"-"` + XMLName xml.Name `xml:"RestoreRequest"` + Days int `xml:"Days"` + Tier RestoreTierType `xml:"GlacierJobParameters>Tier,omitempty"` +} + +// ISseHeader defines the sse encryption header +type ISseHeader interface { + GetEncryption() string + GetKey() string +} + +// SseKmsHeader defines the SseKms header +type SseKmsHeader struct { + Encryption string + Key string + isObs bool +} + +// SseCHeader defines the SseC header +type SseCHeader struct { + Encryption string + Key string + KeyMD5 string +} + +// GetObjectMetadataInput is the input parameter of GetObjectMetadata function +type GetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + Origin string + RequestHeader string + SseHeader ISseHeader +} + +// GetObjectMetadataOutput is the result of GetObjectMetadata function +type GetObjectMetadataOutput struct { + BaseModel + VersionId string + WebsiteRedirectLocation string + Expiration string + Restore string + ObjectType string + NextAppendPosition string + StorageClass StorageClassType + ContentLength int64 + ContentType string + ETag string + AllowOrigin string + AllowHeader string + AllowMethod string + ExposeHeader string + MaxAgeSeconds int + LastModified time.Time + SseHeader ISseHeader + Metadata map[string]string +} + +// GetObjectInput is the input parameter of GetObject function +type GetObjectInput struct { + GetObjectMetadataInput + IfMatch string + IfNoneMatch string + IfUnmodifiedSince time.Time + IfModifiedSince time.Time + RangeStart int64 + RangeEnd int64 + ImageProcess string + ResponseCacheControl string + ResponseContentDisposition string + ResponseContentEncoding string + ResponseContentLanguage string + ResponseContentType string + ResponseExpires string +} + +// GetObjectOutput is the result of GetObject function +type GetObjectOutput struct { + GetObjectMetadataOutput + DeleteMarker bool + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + Expires string + Body io.ReadCloser +} + +// ObjectOperationInput defines the object operation properties +type ObjectOperationInput struct { + Bucket string + Key string + ACL AclType + GrantReadId string + GrantReadAcpId string + GrantWriteAcpId string + GrantFullControlId string + StorageClass StorageClassType + WebsiteRedirectLocation string + Expires int64 + SseHeader ISseHeader + Metadata map[string]string +} + +// PutObjectBasicInput defines the basic object operation properties +type PutObjectBasicInput struct { + ObjectOperationInput + ContentType string + ContentMD5 string + ContentLength int64 +} + +// PutObjectInput is the input parameter of PutObject function +type PutObjectInput struct { + PutObjectBasicInput + Body io.Reader +} + +// PutFileInput is the input parameter of PutFile function +type PutFileInput struct { + PutObjectBasicInput + SourceFile string +} + +// PutObjectOutput is the result of PutObject function +type PutObjectOutput struct { + BaseModel + VersionId string + SseHeader ISseHeader + StorageClass StorageClassType + ETag string +} + +// CopyObjectInput is the input parameter of CopyObject function +type CopyObjectInput struct { + ObjectOperationInput + CopySourceBucket string + CopySourceKey string + CopySourceVersionId string + CopySourceIfMatch string + CopySourceIfNoneMatch string + CopySourceIfUnmodifiedSince time.Time + CopySourceIfModifiedSince time.Time + SourceSseHeader ISseHeader + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + MetadataDirective MetadataDirectiveType + SuccessActionRedirect string +} + +// CopyObjectOutput is the result of CopyObject function +type CopyObjectOutput struct { + BaseModel + CopySourceVersionId string `xml:"-"` + VersionId string `xml:"-"` + SseHeader ISseHeader `xml:"-"` + XMLName xml.Name `xml:"CopyObjectResult"` + LastModified time.Time `xml:"LastModified"` + ETag string `xml:"ETag"` +} + +// AbortMultipartUploadInput is the input parameter of AbortMultipartUpload function +type AbortMultipartUploadInput struct { + Bucket string + Key string + UploadId string +} + +// InitiateMultipartUploadInput is the input parameter of InitiateMultipartUpload function +type InitiateMultipartUploadInput struct { + ObjectOperationInput + ContentType string +} + +// InitiateMultipartUploadOutput is the result of InitiateMultipartUpload function +type InitiateMultipartUploadOutput struct { + BaseModel + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + SseHeader ISseHeader +} + +// UploadPartInput is the input parameter of UploadPart function +type UploadPartInput struct { + Bucket string + Key string + PartNumber int + UploadId string + ContentMD5 string + SseHeader ISseHeader + Body io.Reader + SourceFile string + Offset int64 + PartSize int64 +} + +// UploadPartOutput is the result of UploadPart function +type UploadPartOutput struct { + BaseModel + PartNumber int + ETag string + SseHeader ISseHeader +} + +// Part defines the part properties +type Part struct { + XMLName xml.Name `xml:"Part"` + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified,omitempty"` + Size int64 `xml:"Size,omitempty"` +} + +// CompleteMultipartUploadInput is the input parameter of CompleteMultipartUpload function +type CompleteMultipartUploadInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + UploadId string `xml:"-"` + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []Part `xml:"Part"` +} + +// CompleteMultipartUploadOutput is the result of CompleteMultipartUpload function +type CompleteMultipartUploadOutput struct { + BaseModel + VersionId string `xml:"-"` + SseHeader ISseHeader `xml:"-"` + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +// ListPartsInput is the input parameter of ListParts function +type ListPartsInput struct { + Bucket string + Key string + UploadId string + MaxParts int + PartNumberMarker int +} + +// ListPartsOutput is the result of ListParts function +type ListPartsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListPartsResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + PartNumberMarker int `xml:"PartNumberMarker"` + NextPartNumberMarker int `xml:"NextPartNumberMarker"` + MaxParts int `xml:"MaxParts"` + IsTruncated bool `xml:"IsTruncated"` + StorageClass StorageClassType `xml:"StorageClass"` + Initiator Initiator `xml:"Initiator"` + Owner Owner `xml:"Owner"` + Parts []Part `xml:"Part"` +} + +// CopyPartInput is the input parameter of CopyPart function +type CopyPartInput struct { + Bucket string + Key string + UploadId string + PartNumber int + CopySourceBucket string + CopySourceKey string + CopySourceVersionId string + CopySourceRangeStart int64 + CopySourceRangeEnd int64 + SseHeader ISseHeader + SourceSseHeader ISseHeader +} + +// CopyPartOutput is the result of CopyPart function +type CopyPartOutput struct { + BaseModel + XMLName xml.Name `xml:"CopyPartResult"` + PartNumber int `xml:"-"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified"` + SseHeader ISseHeader `xml:"-"` +} + +// CreateSignedUrlInput is the input parameter of CreateSignedUrl function +type CreateSignedUrlInput struct { + Method HttpMethodType + Bucket string + Key string + SubResource SubResourceType + Expires int + Headers map[string]string + QueryParams map[string]string +} + +// CreateSignedUrlOutput is the result of CreateSignedUrl function +type CreateSignedUrlOutput struct { + SignedUrl string + ActualSignedRequestHeaders http.Header +} + +// CreateBrowserBasedSignatureInput is the input parameter of CreateBrowserBasedSignature function. +type CreateBrowserBasedSignatureInput struct { + Bucket string + Key string + Expires int + FormParams map[string]string +} + +// CreateBrowserBasedSignatureOutput is the result of CreateBrowserBasedSignature function. +type CreateBrowserBasedSignatureOutput struct { + OriginPolicy string + Policy string + Algorithm string + Credential string + Date string + Signature string +} + +// HeadObjectInput is the input parameter of HeadObject function +type HeadObjectInput struct { + Bucket string + Key string + VersionId string +} + +// BucketPayer defines the request payment configuration +type BucketPayer struct { + XMLName xml.Name `xml:"RequestPaymentConfiguration"` + Payer PayerType `xml:"Payer"` +} + +// SetBucketRequestPaymentInput is the input parameter of SetBucketRequestPayment function +type SetBucketRequestPaymentInput struct { + Bucket string `xml:"-"` + BucketPayer +} + +// GetBucketRequestPaymentOutput is the result of GetBucketRequestPayment function +type GetBucketRequestPaymentOutput struct { + BaseModel + BucketPayer +} + +// UploadFileInput is the input parameter of UploadFile function +type UploadFileInput struct { + ObjectOperationInput + ContentType string + UploadFile string + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string +} + +// DownloadFileInput is the input parameter of DownloadFile function +type DownloadFileInput struct { + GetObjectMetadataInput + IfMatch string + IfNoneMatch string + IfModifiedSince time.Time + IfUnmodifiedSince time.Time + DownloadFile string + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string +} + +// SetBucketFetchPolicyInput is the input parameter of SetBucketFetchPolicy function +type SetBucketFetchPolicyInput struct { + Bucket string + Status FetchPolicyStatusType `json:"status"` + Agency string `json:"agency"` +} + +// GetBucketFetchPolicyInput is the input parameter of GetBucketFetchPolicy function +type GetBucketFetchPolicyInput struct { + Bucket string +} + +// GetBucketFetchPolicyOutput is the result of GetBucketFetchPolicy function +type GetBucketFetchPolicyOutput struct { + BaseModel + FetchResponse `json:"fetch"` +} + +// FetchResponse defines the response fetch policy configuration +type FetchResponse struct { + Status FetchPolicyStatusType `json:"status"` + Agency string `json:"agency"` +} + +// DeleteBucketFetchPolicyInput is the input parameter of DeleteBucketFetchPolicy function +type DeleteBucketFetchPolicyInput struct { + Bucket string +} + +// SetBucketFetchJobInput is the input parameter of SetBucketFetchJob function +type SetBucketFetchJobInput struct { + Bucket string `json:"bucket"` + URL string `json:"url"` + Host string `json:"host,omitempty"` + Key string `json:"key,omitempty"` + Md5 string `json:"md5,omitempty"` + CallBackURL string `json:"callbackurl,omitempty"` + CallBackBody string `json:"callbackbody,omitempty"` + CallBackBodyType string `json:"callbackbodytype,omitempty"` + CallBackHost string `json:"callbackhost,omitempty"` + FileType string `json:"file_type,omitempty"` + IgnoreSameKey bool `json:"ignore_same_key,omitempty"` + ObjectHeaders map[string]string `json:"objectheaders,omitempty"` + Etag string `json:"etag,omitempty"` + TrustName string `json:"trustname,omitempty"` +} + +// SetBucketFetchJobOutput is the result of SetBucketFetchJob function +type SetBucketFetchJobOutput struct { + BaseModel + SetBucketFetchJobResponse +} + +// SetBucketFetchJobResponse defines the response SetBucketFetchJob configuration +type SetBucketFetchJobResponse struct { + ID string `json:"id"` + Wait int `json:"Wait"` +} + +// GetBucketFetchJobInput is the input parameter of GetBucketFetchJob function +type GetBucketFetchJobInput struct { + Bucket string + JobID string +} + +// GetBucketFetchJobOutput is the result of GetBucketFetchJob function +type GetBucketFetchJobOutput struct { + BaseModel + GetBucketFetchJobResponse +} + +// GetBucketFetchJobResponse defines the response fetch job configuration +type GetBucketFetchJobResponse struct { + Err string `json:"err"` + Code string `json:"code"` + Status string `json:"status"` + Job JobResponse `json:"job"` +} + +// JobResponse defines the response job configuration +type JobResponse struct { + Bucket string `json:"bucket"` + URL string `json:"url"` + Host string `json:"host"` + Key string `json:"key"` + Md5 string `json:"md5"` + CallBackURL string `json:"callbackurl"` + CallBackBody string `json:"callbackbody"` + CallBackBodyType string `json:"callbackbodytype"` + CallBackHost string `json:"callbackhost"` + FileType string `json:"file_type"` + IgnoreSameKey bool `json:"ignore_same_key"` +} diff --git a/cve-vulner-manager/obs/pool.go b/cve-vulner-manager/obs/pool.go new file mode 100644 index 0000000000000000000000000000000000000000..1755c96ac5129a42076ab2a5763bb72208599b05 --- /dev/null +++ b/cve-vulner-manager/obs/pool.go @@ -0,0 +1,542 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +//nolint:structcheck, unused +package obs + +import ( + "errors" + "fmt" + "runtime" + "sync" + "sync/atomic" + "time" +) + +// Future defines interface with function: Get +type Future interface { + Get() interface{} +} + +// FutureResult for task result +type FutureResult struct { + result interface{} + resultChan chan interface{} + lock sync.Mutex +} + +type panicResult struct { + presult interface{} +} + +func (f *FutureResult) checkPanic() interface{} { + if r, ok := f.result.(panicResult); ok { + panic(r.presult) + } + return f.result +} + +// Get gets the task result +func (f *FutureResult) Get() interface{} { + if f.resultChan == nil { + return f.checkPanic() + } + f.lock.Lock() + defer f.lock.Unlock() + if f.resultChan == nil { + return f.checkPanic() + } + + f.result = <-f.resultChan + close(f.resultChan) + f.resultChan = nil + return f.checkPanic() +} + +// Task defines interface with function: Run +type Task interface { + Run() interface{} +} + +type funcWrapper struct { + f func() interface{} +} + +func (fw *funcWrapper) Run() interface{} { + if fw.f != nil { + return fw.f() + } + return nil +} + +type taskWrapper struct { + t Task + f *FutureResult +} + +func (tw *taskWrapper) Run() interface{} { + if tw.t != nil { + return tw.t.Run() + } + return nil +} + +type signalTask struct { + id string +} + +func (signalTask) Run() interface{} { + return nil +} + +type worker struct { + name string + taskQueue chan Task + wg *sync.WaitGroup + pool *RoutinePool +} + +func runTask(t Task) { + if tw, ok := t.(*taskWrapper); ok { + defer func() { + if r := recover(); r != nil { + tw.f.resultChan <- panicResult{ + presult: r, + } + } + }() + ret := t.Run() + tw.f.resultChan <- ret + } else { + t.Run() + } +} + +func (*worker) runTask(t Task) { + runTask(t) +} + +func (w *worker) start() { + go func() { + defer func() { + if w.wg != nil { + w.wg.Done() + } + }() + for { + task, ok := <-w.taskQueue + if !ok { + break + } + w.pool.AddCurrentWorkingCnt(1) + w.runTask(task) + w.pool.AddCurrentWorkingCnt(-1) + if w.pool.autoTuneWorker(w) { + break + } + } + }() +} + +func (w *worker) release() { + w.taskQueue = nil + w.wg = nil + w.pool = nil +} + +// Pool defines coroutine pool interface +type Pool interface { + ShutDown() + Submit(t Task) (Future, error) + SubmitFunc(f func() interface{}) (Future, error) + Execute(t Task) + ExecuteFunc(f func() interface{}) + GetMaxWorkerCnt() int64 + AddMaxWorkerCnt(value int64) int64 + GetCurrentWorkingCnt() int64 + AddCurrentWorkingCnt(value int64) int64 + GetWorkerCnt() int64 + AddWorkerCnt(value int64) int64 + EnableAutoTune() +} + +type basicPool struct { + maxWorkerCnt int64 + workerCnt int64 + currentWorkingCnt int64 + isShutDown int32 +} + +// ErrTaskInvalid will be returned if the task is nil +var ErrTaskInvalid = errors.New("Task is nil") + +func (pool *basicPool) GetCurrentWorkingCnt() int64 { + return atomic.LoadInt64(&pool.currentWorkingCnt) +} + +func (pool *basicPool) AddCurrentWorkingCnt(value int64) int64 { + return atomic.AddInt64(&pool.currentWorkingCnt, value) +} + +func (pool *basicPool) GetWorkerCnt() int64 { + return atomic.LoadInt64(&pool.workerCnt) +} + +func (pool *basicPool) AddWorkerCnt(value int64) int64 { + return atomic.AddInt64(&pool.workerCnt, value) +} + +func (pool *basicPool) GetMaxWorkerCnt() int64 { + return atomic.LoadInt64(&pool.maxWorkerCnt) +} + +func (pool *basicPool) AddMaxWorkerCnt(value int64) int64 { + return atomic.AddInt64(&pool.maxWorkerCnt, value) +} + +func (pool *basicPool) CompareAndSwapCurrentWorkingCnt(oldValue, newValue int64) bool { + return atomic.CompareAndSwapInt64(&pool.currentWorkingCnt, oldValue, newValue) +} + +func (pool *basicPool) EnableAutoTune() { + +} + +// RoutinePool defines the coroutine pool struct +type RoutinePool struct { + basicPool + taskQueue chan Task + dispatchQueue chan Task + workers map[string]*worker + cacheCnt int + wg *sync.WaitGroup + lock *sync.Mutex + shutDownWg *sync.WaitGroup + autoTune int32 +} + +// ErrSubmitTimeout will be returned if submit task timeout when calling SubmitWithTimeout function +var ErrSubmitTimeout = errors.New("Submit task timeout") + +// ErrPoolShutDown will be returned if RoutinePool is shutdown +var ErrPoolShutDown = errors.New("RoutinePool is shutdown") + +// ErrTaskReject will be returned if submit task is rejected +var ErrTaskReject = errors.New("Submit task is rejected") + +var closeQueue = signalTask{id: "closeQueue"} + +// NewRoutinePool creates a RoutinePool instance +func NewRoutinePool(maxWorkerCnt, cacheCnt int) Pool { + if maxWorkerCnt <= 0 { + maxWorkerCnt = runtime.NumCPU() + } + + pool := &RoutinePool{ + cacheCnt: cacheCnt, + wg: new(sync.WaitGroup), + lock: new(sync.Mutex), + shutDownWg: new(sync.WaitGroup), + autoTune: 0, + } + pool.isShutDown = 0 + pool.maxWorkerCnt += int64(maxWorkerCnt) + if pool.cacheCnt <= 0 { + pool.taskQueue = make(chan Task) + } else { + pool.taskQueue = make(chan Task, pool.cacheCnt) + } + pool.workers = make(map[string]*worker, pool.maxWorkerCnt) + // dispatchQueue must not have length + pool.dispatchQueue = make(chan Task) + pool.dispatcher() + + return pool +} + +// EnableAutoTune sets the autoTune enabled +func (pool *RoutinePool) EnableAutoTune() { + atomic.StoreInt32(&pool.autoTune, 1) +} + +func (pool *RoutinePool) checkStatus(t Task) error { + if t == nil { + return ErrTaskInvalid + } + + if atomic.LoadInt32(&pool.isShutDown) == 1 { + return ErrPoolShutDown + } + return nil +} + +func (pool *RoutinePool) dispatcher() { + pool.shutDownWg.Add(1) + go func() { + for { + task, ok := <-pool.dispatchQueue + if !ok { + break + } + + if task == closeQueue { + close(pool.taskQueue) + pool.shutDownWg.Done() + continue + } + + if pool.GetWorkerCnt() < pool.GetMaxWorkerCnt() { + pool.addWorker() + } + + pool.taskQueue <- task + } + }() +} + +// AddMaxWorkerCnt sets the maxWorkerCnt field's value and returns it +func (pool *RoutinePool) AddMaxWorkerCnt(value int64) int64 { + if atomic.LoadInt32(&pool.autoTune) == 1 { + return pool.basicPool.AddMaxWorkerCnt(value) + } + return pool.GetMaxWorkerCnt() +} + +func (pool *RoutinePool) addWorker() { + if atomic.LoadInt32(&pool.autoTune) == 1 { + pool.lock.Lock() + defer pool.lock.Unlock() + } + w := &worker{} + w.name = fmt.Sprintf("woker-%d", len(pool.workers)) + w.taskQueue = pool.taskQueue + w.wg = pool.wg + pool.AddWorkerCnt(1) + w.pool = pool + pool.workers[w.name] = w + pool.wg.Add(1) + w.start() +} + +func (pool *RoutinePool) autoTuneWorker(w *worker) bool { + if atomic.LoadInt32(&pool.autoTune) == 0 { + return false + } + + if w == nil { + return false + } + + workerCnt := pool.GetWorkerCnt() + maxWorkerCnt := pool.GetMaxWorkerCnt() + if workerCnt > maxWorkerCnt && atomic.CompareAndSwapInt64(&pool.workerCnt, workerCnt, workerCnt-1) { + pool.lock.Lock() + defer pool.lock.Unlock() + delete(pool.workers, w.name) + w.wg.Done() + w.release() + return true + } + + return false +} + +// ExecuteFunc creates a funcWrapper instance with the specified function and calls the Execute function +func (pool *RoutinePool) ExecuteFunc(f func() interface{}) { + fw := &funcWrapper{ + f: f, + } + pool.Execute(fw) +} + +// Execute pushes the specified task to the dispatchQueue +func (pool *RoutinePool) Execute(t Task) { + if t != nil { + pool.dispatchQueue <- t + } +} + +// SubmitFunc creates a funcWrapper instance with the specified function and calls the Submit function +func (pool *RoutinePool) SubmitFunc(f func() interface{}) (Future, error) { + fw := &funcWrapper{ + f: f, + } + return pool.Submit(fw) +} + +// Submit pushes the specified task to the dispatchQueue, and returns the FutureResult and error info +func (pool *RoutinePool) Submit(t Task) (Future, error) { + if err := pool.checkStatus(t); err != nil { + return nil, err + } + f := &FutureResult{} + f.resultChan = make(chan interface{}, 1) + tw := &taskWrapper{ + t: t, + f: f, + } + pool.dispatchQueue <- tw + return f, nil +} + +// SubmitWithTimeout pushes the specified task to the dispatchQueue, and returns the FutureResult and error info. +// Also takes a timeout value, will return ErrSubmitTimeout if it does't complete within that time. +func (pool *RoutinePool) SubmitWithTimeout(t Task, timeout int64) (Future, error) { + if timeout <= 0 { + return pool.Submit(t) + } + if err := pool.checkStatus(t); err != nil { + return nil, err + } + timeoutChan := make(chan bool, 1) + go func() { + time.Sleep(time.Duration(time.Millisecond * time.Duration(timeout))) + timeoutChan <- true + close(timeoutChan) + }() + + f := &FutureResult{} + f.resultChan = make(chan interface{}, 1) + tw := &taskWrapper{ + t: t, + f: f, + } + select { + case pool.dispatchQueue <- tw: + return f, nil + case _, ok := <-timeoutChan: + if ok { + return nil, ErrSubmitTimeout + } + return nil, ErrSubmitTimeout + } +} + +func (pool *RoutinePool) beforeCloseDispatchQueue() { + if !atomic.CompareAndSwapInt32(&pool.isShutDown, 0, 1) { + return + } + pool.dispatchQueue <- closeQueue + pool.wg.Wait() +} + +func (pool *RoutinePool) doCloseDispatchQueue() { + close(pool.dispatchQueue) + pool.shutDownWg.Wait() +} + +// ShutDown closes the RoutinePool instance +func (pool *RoutinePool) ShutDown() { + pool.beforeCloseDispatchQueue() + pool.doCloseDispatchQueue() + for _, w := range pool.workers { + w.release() + } + pool.workers = nil + pool.taskQueue = nil + pool.dispatchQueue = nil +} + +// NoChanPool defines the coroutine pool struct +type NoChanPool struct { + basicPool + wg *sync.WaitGroup + tokens chan interface{} +} + +// NewNochanPool creates a new NoChanPool instance +func NewNochanPool(maxWorkerCnt int) Pool { + if maxWorkerCnt <= 0 { + maxWorkerCnt = runtime.NumCPU() + } + + pool := &NoChanPool{ + wg: new(sync.WaitGroup), + tokens: make(chan interface{}, maxWorkerCnt), + } + pool.isShutDown = 0 + pool.AddMaxWorkerCnt(int64(maxWorkerCnt)) + + for i := 0; i < maxWorkerCnt; i++ { + pool.tokens <- struct{}{} + } + + return pool +} + +func (pool *NoChanPool) acquire() { + <-pool.tokens +} + +func (pool *NoChanPool) release() { + pool.tokens <- 1 +} + +func (pool *NoChanPool) execute(t Task) { + pool.wg.Add(1) + go func() { + pool.acquire() + defer func() { + pool.release() + pool.wg.Done() + }() + runTask(t) + }() +} + +// ShutDown closes the NoChanPool instance +func (pool *NoChanPool) ShutDown() { + if !atomic.CompareAndSwapInt32(&pool.isShutDown, 0, 1) { + return + } + pool.wg.Wait() +} + +// Execute executes the specified task +func (pool *NoChanPool) Execute(t Task) { + if t != nil { + pool.execute(t) + } +} + +// ExecuteFunc creates a funcWrapper instance with the specified function and calls the Execute function +func (pool *NoChanPool) ExecuteFunc(f func() interface{}) { + fw := &funcWrapper{ + f: f, + } + pool.Execute(fw) +} + +// Submit executes the specified task, and returns the FutureResult and error info +func (pool *NoChanPool) Submit(t Task) (Future, error) { + if t == nil { + return nil, ErrTaskInvalid + } + + f := &FutureResult{} + f.resultChan = make(chan interface{}, 1) + tw := &taskWrapper{ + t: t, + f: f, + } + + pool.execute(tw) + return f, nil +} + +// SubmitFunc creates a funcWrapper instance with the specified function and calls the Submit function +func (pool *NoChanPool) SubmitFunc(f func() interface{}) (Future, error) { + fw := &funcWrapper{ + f: f, + } + return pool.Submit(fw) +} diff --git a/cve-vulner-manager/obs/provider.go b/cve-vulner-manager/obs/provider.go new file mode 100644 index 0000000000000000000000000000000000000000..2e485c1134bdf5f546cd63f495b4b14cbf0e81fb --- /dev/null +++ b/cve-vulner-manager/obs/provider.go @@ -0,0 +1,243 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "encoding/json" + "io/ioutil" + "math/rand" + "net" + "net/http" + "os" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + accessKeyEnv = "OBS_ACCESS_KEY_ID" + securityKeyEnv = "OBS_SECRET_ACCESS_KEY" + securityTokenEnv = "OBS_SECURITY_TOKEN" + ecsRequestURL = "http://169.254.169.254/openstack/latest/securitykey" +) + +type securityHolder struct { + ak string + sk string + securityToken string +} + +var emptySecurityHolder = securityHolder{} + +type securityProvider interface { + getSecurity() securityHolder +} + +type BasicSecurityProvider struct { + val atomic.Value +} + +func (bsp *BasicSecurityProvider) getSecurity() securityHolder { + if sh, ok := bsp.val.Load().(securityHolder); ok { + return sh + } + return emptySecurityHolder +} + +func (bsp *BasicSecurityProvider) refresh(ak, sk, securityToken string) { + bsp.val.Store(securityHolder{ak: strings.TrimSpace(ak), sk: strings.TrimSpace(sk), securityToken: strings.TrimSpace(securityToken)}) +} + +func NewBasicSecurityProvider(ak, sk, securityToken string) *BasicSecurityProvider { + bsp := &BasicSecurityProvider{} + bsp.refresh(ak, sk, securityToken) + return bsp +} + +type EnvSecurityProvider struct { + sh securityHolder + suffix string + once sync.Once +} + +func (esp *EnvSecurityProvider) getSecurity() securityHolder { + //ensure run only once + esp.once.Do(func() { + esp.sh = securityHolder{ + ak: strings.TrimSpace(os.Getenv(accessKeyEnv + esp.suffix)), + sk: strings.TrimSpace(os.Getenv(securityKeyEnv + esp.suffix)), + securityToken: strings.TrimSpace(os.Getenv(securityTokenEnv + esp.suffix)), + } + }) + + return esp.sh +} + +func NewEnvSecurityProvider(suffix string) *EnvSecurityProvider { + if suffix != "" { + suffix = "_" + suffix + } + esp := &EnvSecurityProvider{ + suffix: suffix, + } + return esp +} + +type TemporarySecurityHolder struct { + securityHolder + expireDate time.Time +} + +var emptyTemporarySecurityHolder = TemporarySecurityHolder{} + +type EcsSecurityProvider struct { + val atomic.Value + lock sync.Mutex + httpClient *http.Client + prefetch int32 + retryCount int +} + +func (ecsSp *EcsSecurityProvider) loadTemporarySecurityHolder() (TemporarySecurityHolder, bool) { + if sh := ecsSp.val.Load(); sh == nil { + return emptyTemporarySecurityHolder, false + } else if _sh, ok := sh.(TemporarySecurityHolder); !ok { + return emptyTemporarySecurityHolder, false + } else { + return _sh, true + } +} + +func (ecsSp *EcsSecurityProvider) getAndSetSecurityWithOutLock() securityHolder { + _sh := TemporarySecurityHolder{} + _sh.expireDate = time.Now().Add(time.Minute * 5) + retryCount := 0 + for { + if req, err := http.NewRequest("GET", ecsRequestURL, nil); err == nil { + start := GetCurrentTimestamp() + res, err := ecsSp.httpClient.Do(req) + if err == nil { + if data, _err := ioutil.ReadAll(res.Body); _err == nil { + temp := &struct { + Credential struct { + AK string `json:"access,omitempty"` + SK string `json:"secret,omitempty"` + SecurityToken string `json:"securitytoken,omitempty"` + ExpireDate time.Time `json:"expires_at,omitempty"` + } `json:"credential"` + }{} + + doLog(LEVEL_DEBUG, "Get the json data from ecs succeed") + + if jsonErr := json.Unmarshal(data, temp); jsonErr == nil { + _sh.ak = temp.Credential.AK + _sh.sk = temp.Credential.SK + _sh.securityToken = temp.Credential.SecurityToken + _sh.expireDate = temp.Credential.ExpireDate.Add(time.Minute * -1) + + doLog(LEVEL_INFO, "Get security from ecs succeed, AK:xxxx, SK:xxxx, SecurityToken:xxxx, ExprireDate %s", _sh.expireDate) + + doLog(LEVEL_INFO, "Get security from ecs succeed, cost %d ms", (GetCurrentTimestamp() - start)) + break + } else { + err = jsonErr + } + } else { + err = _err + } + } + + doLog(LEVEL_WARN, "Try to get security from ecs failed, cost %d ms, err %s", (GetCurrentTimestamp() - start), err.Error()) + } + + if retryCount >= ecsSp.retryCount { + doLog(LEVEL_WARN, "Try to get security from ecs failed and exceed the max retry count") + break + } + sleepTime := float64(retryCount+2) * rand.Float64() + if sleepTime > 10 { + sleepTime = 10 + } + time.Sleep(time.Duration(sleepTime * float64(time.Second))) + retryCount++ + } + + ecsSp.val.Store(_sh) + return _sh.securityHolder +} + +func (ecsSp *EcsSecurityProvider) getAndSetSecurity() securityHolder { + ecsSp.lock.Lock() + defer ecsSp.lock.Unlock() + tsh, succeed := ecsSp.loadTemporarySecurityHolder() + if !succeed || time.Now().After(tsh.expireDate) { + return ecsSp.getAndSetSecurityWithOutLock() + } + return tsh.securityHolder +} + +func (ecsSp *EcsSecurityProvider) getSecurity() securityHolder { + if tsh, succeed := ecsSp.loadTemporarySecurityHolder(); succeed { + if time.Now().Before(tsh.expireDate) { + //not expire + if time.Now().Add(time.Minute*5).After(tsh.expireDate) && atomic.CompareAndSwapInt32(&ecsSp.prefetch, 0, 1) { + //do prefetch + sh := ecsSp.getAndSetSecurityWithOutLock() + atomic.CompareAndSwapInt32(&ecsSp.prefetch, 1, 0) + return sh + } + return tsh.securityHolder + } + return ecsSp.getAndSetSecurity() + } + + return ecsSp.getAndSetSecurity() +} + +func getInternalTransport() *http.Transport { + + timeout := 10 + transport := &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + start := GetCurrentTimestamp() + conn, err := (&net.Dialer{ + Timeout: time.Second * time.Duration(timeout), + Resolver: net.DefaultResolver, + }).Dial(network, addr) + + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http dial cost %d ms", (GetCurrentTimestamp() - start)) + } + if err != nil { + return nil, err + } + return getConnDelegate(conn, timeout, timeout*10), nil + }, + MaxIdleConns: 10, + MaxIdleConnsPerHost: 10, + ResponseHeaderTimeout: time.Second * time.Duration(timeout), + IdleConnTimeout: time.Second * time.Duration(DEFAULT_IDLE_CONN_TIMEOUT), + DisableCompression: true, + } + + return transport +} + +func NewEcsSecurityProvider(retryCount int) *EcsSecurityProvider { + ecsSp := &EcsSecurityProvider{ + retryCount: retryCount, + } + ecsSp.httpClient = &http.Client{Transport: getInternalTransport(), CheckRedirect: checkRedirectFunc} + return ecsSp +} diff --git a/cve-vulner-manager/obs/temporary.go b/cve-vulner-manager/obs/temporary.go new file mode 100644 index 0000000000000000000000000000000000000000..b17e68cc1de2eb639545bfba7626055947172570 --- /dev/null +++ b/cve-vulner-manager/obs/temporary.go @@ -0,0 +1,790 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +// CreateSignedUrl creates signed url with the specified CreateSignedUrlInput, and returns the CreateSignedUrlOutput and error +func (obsClient ObsClient) CreateSignedUrl(input *CreateSignedUrlInput) (output *CreateSignedUrlOutput, err error) { + if input == nil { + return nil, errors.New("CreateSignedUrlInput is nil") + } + + params := make(map[string]string, len(input.QueryParams)) + for key, value := range input.QueryParams { + params[key] = value + } + + if input.SubResource != "" { + params[string(input.SubResource)] = "" + } + + headers := make(map[string][]string, len(input.Headers)) + for key, value := range input.Headers { + headers[key] = []string{value} + } + + if input.Expires <= 0 { + input.Expires = 300 + } + + requestURL, err := obsClient.doAuthTemporary(string(input.Method), input.Bucket, input.Key, params, headers, int64(input.Expires)) + if err != nil { + return nil, err + } + + output = &CreateSignedUrlOutput{ + SignedUrl: requestURL, + ActualSignedRequestHeaders: headers, + } + return +} + +func (obsClient ObsClient) isSecurityToken(params map[string]string, sh securityHolder) { + if sh.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = sh.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = sh.securityToken + } + } +} + +// CreateBrowserBasedSignature gets the browser based signature with the specified CreateBrowserBasedSignatureInput, +// and returns the CreateBrowserBasedSignatureOutput and error +func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBasedSignatureInput) (output *CreateBrowserBasedSignatureOutput, err error) { + if input == nil { + return nil, errors.New("CreateBrowserBasedSignatureInput is nil") + } + + params := make(map[string]string, len(input.FormParams)) + for key, value := range input.FormParams { + params[key] = value + } + + date := time.Now().UTC() + shortDate := date.Format(SHORT_DATE_FORMAT) + longDate := date.Format(LONG_DATE_FORMAT) + sh := obsClient.getSecurity() + + credential, _ := getCredential(sh.ak, obsClient.conf.region, shortDate) + + if input.Expires <= 0 { + input.Expires = 300 + } + + expiration := date.Add(time.Second * time.Duration(input.Expires)).Format(ISO8601_DATE_FORMAT) + if obsClient.conf.signature == SignatureV4 { + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate + } + + obsClient.isSecurityToken(params, sh) + + matchAnyBucket := true + matchAnyKey := true + count := 5 + if bucket := strings.TrimSpace(input.Bucket); bucket != "" { + params["bucket"] = bucket + matchAnyBucket = false + count-- + } + + if key := strings.TrimSpace(input.Key); key != "" { + params["key"] = key + matchAnyKey = false + count-- + } + + originPolicySlice := make([]string, 0, len(params)+count) + originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"expiration\":\"%s\",", expiration)) + originPolicySlice = append(originPolicySlice, "\"conditions\":[") + for key, value := range params { + if _key := strings.TrimSpace(strings.ToLower(key)); _key != "" { + originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"%s\":\"%s\"},", _key, value)) + } + } + + if matchAnyBucket { + originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$bucket\", \"\"],") + } + + if matchAnyKey { + originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$key\", \"\"],") + } + + originPolicySlice = append(originPolicySlice, "]}") + + originPolicy := strings.Join(originPolicySlice, "") + policy := Base64Encode([]byte(originPolicy)) + var signature string + if obsClient.conf.signature == SignatureV4 { + signature = getSignature(policy, sh.sk, obsClient.conf.region, shortDate) + } else { + signature = Base64Encode(HmacSha1([]byte(sh.sk), []byte(policy))) + } + + output = &CreateBrowserBasedSignatureOutput{ + OriginPolicy: originPolicy, + Policy: policy, + Algorithm: params[PARAM_ALGORITHM_AMZ_CAMEL], + Credential: params[PARAM_CREDENTIAL_AMZ_CAMEL], + Date: params[PARAM_DATE_AMZ_CAMEL], + Signature: signature, + } + return +} + +// ListBucketsWithSignedUrl lists buckets with the specified signed url and signed request headers +func (obsClient ObsClient) ListBucketsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListBucketsOutput, err error) { + output = &ListBucketsOutput{} + err = obsClient.doHTTPWithSignedURL("ListBuckets", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// CreateBucketWithSignedUrl creates bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) CreateBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("CreateBucket", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketWithSignedUrl deletes bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucket", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketStoragePolicyWithSignedUrl sets bucket storage class with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketStoragePolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketStoragePolicyWithSignedUrl gets bucket storage class with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketStoragePolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// ListObjectsWithSignedUrl lists objects in a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) ListObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListObjectsOutput, err error) { + output = &ListObjectsOutput{} + err = obsClient.doHTTPWithSignedURL("ListObjects", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +// ListVersionsWithSignedUrl lists versioning objects in a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) ListVersionsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListVersionsOutput, err error) { + output = &ListVersionsOutput{} + err = obsClient.doHTTPWithSignedURL("ListVersions", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +// ListMultipartUploadsWithSignedUrl lists the multipart uploads that are initialized but not combined or aborted in a +// specified bucket with the specified signed url and signed request headers +func (obsClient ObsClient) ListMultipartUploadsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListMultipartUploadsOutput, err error) { + output = &ListMultipartUploadsOutput{} + err = obsClient.doHTTPWithSignedURL("ListMultipartUploads", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketQuotaWithSignedUrl sets the bucket quota with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketQuota", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketQuotaWithSignedUrl gets the bucket quota with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketQuotaOutput, err error) { + output = &GetBucketQuotaOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketQuota", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// HeadBucketWithSignedUrl checks whether a bucket exists with the specified signed url and signed request headers +func (obsClient ObsClient) HeadBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("HeadBucket", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// HeadObjectWithSignedUrl checks whether an object exists with the specified signed url and signed request headers +func (obsClient ObsClient) HeadObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("HeadObject", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketMetadataWithSignedUrl gets the metadata of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketMetadataOutput, err error) { + output = &GetBucketMetadataOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetBucketMetadataOutput(output) + } + return +} + +// GetBucketStorageInfoWithSignedUrl gets storage information about a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketStorageInfoWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStorageInfoOutput, err error) { + output = &GetBucketStorageInfoOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketStorageInfo", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketLocationWithSignedUrl gets the location of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketLocationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketLocation", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketAclWithSignedUrl sets the bucket ACL with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketAclWithSignedUrl gets the bucket ACL with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketPolicyWithSignedUrl sets the bucket policy with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketPolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketPolicyWithSignedUrl gets the bucket policy with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketPolicyOutput, err error) { + output = &GetBucketPolicyOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketPolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, false) + if err != nil { + output = nil + } + return +} + +// DeleteBucketPolicyWithSignedUrl deletes the bucket policy with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketPolicy", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketCorsWithSignedUrl sets CORS rules for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketCors", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketCorsWithSignedUrl gets CORS rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketCorsOutput, err error) { + output = &GetBucketCorsOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketCors", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketCorsWithSignedUrl deletes CORS rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketCors", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketVersioningWithSignedUrl sets the versioning status for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketVersioning", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketVersioningWithSignedUrl gets the versioning status of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketVersioningOutput, err error) { + output = &GetBucketVersioningOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketVersioning", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketWebsiteConfigurationWithSignedUrl sets website hosting for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketWebsiteConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketWebsiteConfigurationWithSignedUrl gets the website hosting settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketWebsiteConfigurationOutput, err error) { + output = &GetBucketWebsiteConfigurationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketWebsiteConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketWebsiteConfigurationWithSignedUrl deletes the website hosting settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketWebsiteConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketLoggingConfigurationWithSignedUrl sets the bucket logging with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketLoggingConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketLoggingConfigurationWithSignedUrl gets the logging settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLoggingConfigurationOutput, err error) { + output = &GetBucketLoggingConfigurationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketLoggingConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketLifecycleConfigurationWithSignedUrl sets lifecycle rules for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketLifecycleConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketLifecycleConfigurationWithSignedUrl gets lifecycle rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLifecycleConfigurationOutput, err error) { + output = &GetBucketLifecycleConfigurationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketLifecycleConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketLifecycleConfigurationWithSignedUrl deletes lifecycle rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketLifecycleConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketTaggingWithSignedUrl sets bucket tags with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketTagging", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketTaggingWithSignedUrl gets bucket tags with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketTaggingOutput, err error) { + output = &GetBucketTaggingOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketTagging", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketTaggingWithSignedUrl deletes bucket tags with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketTagging", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketNotificationWithSignedUrl sets event notification for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketNotification", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketNotificationWithSignedUrl gets event notification settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketNotificationOutput, err error) { + output = &GetBucketNotificationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketNotification", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteObjectWithSignedUrl deletes an object with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *DeleteObjectOutput, err error) { + output = &DeleteObjectOutput{} + err = obsClient.doHTTPWithSignedURL("DeleteObject", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseDeleteObjectOutput(output) + } + return +} + +// DeleteObjectsWithSignedUrl deletes objects in a batch with the specified signed url and signed request headers and data +func (obsClient ObsClient) DeleteObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *DeleteObjectsOutput, err error) { + output = &DeleteObjectsOutput{} + err = obsClient.doHTTPWithSignedURL("DeleteObjects", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// SetObjectAclWithSignedUrl sets ACL for an object with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetObjectAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetObjectAclWithSignedUrl gets the ACL of an object with the specified signed url and signed request headers +func (obsClient ObsClient) GetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectAclOutput, err error) { + output = &GetObjectAclOutput{} + err = obsClient.doHTTPWithSignedURL("GetObjectAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if versionID, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionID[0] + } + } + return +} + +// RestoreObjectWithSignedUrl restores an object with the specified signed url and signed request headers and data +func (obsClient ObsClient) RestoreObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("RestoreObject", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetObjectMetadataWithSignedUrl gets object metadata with the specified signed url and signed request headers +func (obsClient ObsClient) GetObjectMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectMetadataOutput, err error) { + output = &GetObjectMetadataOutput{} + err = obsClient.doHTTPWithSignedURL("GetObjectMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetObjectMetadataOutput(output) + } + return +} + +// GetObjectWithSignedUrl downloads object with the specified signed url and signed request headers +func (obsClient ObsClient) GetObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectOutput, err error) { + output = &GetObjectOutput{} + err = obsClient.doHTTPWithSignedURL("GetObject", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetObjectOutput(output) + } + return +} + +// PutObjectWithSignedUrl uploads an object to the specified bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) PutObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *PutObjectOutput, err error) { + output = &PutObjectOutput{} + err = obsClient.doHTTPWithSignedURL("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +// PutFileWithSignedUrl uploads a file to the specified bucket with the specified signed url and signed request headers and sourceFile path +func (obsClient ObsClient) PutFileWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, sourceFile string) (output *PutObjectOutput, err error) { + var data io.Reader + sourceFile = strings.TrimSpace(sourceFile) + if sourceFile != "" { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() + + stat, _err := fd.Stat() + if _err != nil { + err = _err + return nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + + var contentLength int64 + if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH_CAMEL]; ok { + contentLength = StringToInt64(value[0], -1) + } else if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH]; ok { + contentLength = StringToInt64(value[0], -1) + } else { + contentLength = stat.Size() + } + if contentLength > stat.Size() { + return nil, errors.New("ContentLength is larger than fileSize") + } + fileReaderWrapper.totalCount = contentLength + data = fileReaderWrapper + } + + output = &PutObjectOutput{} + err = obsClient.doHTTPWithSignedURL("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +// CopyObjectWithSignedUrl creates a copy for an existing object with the specified signed url and signed request headers +func (obsClient ObsClient) CopyObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyObjectOutput, err error) { + output = &CopyObjectOutput{} + err = obsClient.doHTTPWithSignedURL("CopyObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseCopyObjectOutput(output) + } + return +} + +// AbortMultipartUploadWithSignedUrl aborts a multipart upload in a specified bucket by using the multipart upload ID with the specified signed url and signed request headers +func (obsClient ObsClient) AbortMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("AbortMultipartUpload", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// InitiateMultipartUploadWithSignedUrl initializes a multipart upload with the specified signed url and signed request headers +func (obsClient ObsClient) InitiateMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *InitiateMultipartUploadOutput, err error) { + output = &InitiateMultipartUploadOutput{} + err = obsClient.doHTTPWithSignedURL("InitiateMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseInitiateMultipartUploadOutput(output) + } + return +} + +// UploadPartWithSignedUrl uploads a part to a specified bucket by using a specified multipart upload ID +// with the specified signed url and signed request headers and data +func (obsClient ObsClient) UploadPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *UploadPartOutput, err error) { + output = &UploadPartOutput{} + err = obsClient.doHTTPWithSignedURL("UploadPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseUploadPartOutput(output) + } + return +} + +// CompleteMultipartUploadWithSignedUrl combines the uploaded parts in a specified bucket by using the multipart upload ID +// with the specified signed url and signed request headers and data +func (obsClient ObsClient) CompleteMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *CompleteMultipartUploadOutput, err error) { + output = &CompleteMultipartUploadOutput{} + err = obsClient.doHTTPWithSignedURL("CompleteMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseCompleteMultipartUploadOutput(output) + } + return +} + +// ListPartsWithSignedUrl lists the uploaded parts in a bucket by using the multipart upload ID with the specified signed url and signed request headers +func (obsClient ObsClient) ListPartsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListPartsOutput, err error) { + output = &ListPartsOutput{} + err = obsClient.doHTTPWithSignedURL("ListParts", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// CopyPartWithSignedUrl copy a part to a specified bucket by using a specified multipart upload ID with the specified signed url and signed request headers +func (obsClient ObsClient) CopyPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyPartOutput, err error) { + output = &CopyPartOutput{} + err = obsClient.doHTTPWithSignedURL("CopyPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseCopyPartOutput(output) + } + return +} + +// SetBucketRequestPaymentWithSignedUrl sets requester-pays setting for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketRequestPaymentWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketRequestPayment", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketRequestPaymentWithSignedUrl gets requester-pays setting of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketRequestPaymentWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketRequestPaymentOutput, err error) { + output = &GetBucketRequestPaymentOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketRequestPayment", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} diff --git a/cve-vulner-manager/obs/trait.go b/cve-vulner-manager/obs/trait.go new file mode 100644 index 0000000000000000000000000000000000000000..ddacf872ea2a7db24bba7b63c73665e1894183a5 --- /dev/null +++ b/cve-vulner-manager/obs/trait.go @@ -0,0 +1,912 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +//nolint:structcheck, unused +package obs + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" +) + +// IReadCloser defines interface with function: setReadCloser +type IReadCloser interface { + setReadCloser(body io.ReadCloser) +} + +func (output *GetObjectOutput) setReadCloser(body io.ReadCloser) { + output.Body = body +} + +func setHeaders(headers map[string][]string, header string, headerValue []string, isObs bool) { + if isObs { + header = HEADER_PREFIX_OBS + header + headers[header] = headerValue + } else { + header = HEADER_PREFIX + header + headers[header] = headerValue + } +} + +func setHeadersNext(headers map[string][]string, header string, headerNext string, headerValue []string, isObs bool) { + if isObs { + headers[header] = headerValue + } else { + headers[headerNext] = headerValue + } +} + +// IBaseModel defines interface for base response model +type IBaseModel interface { + setStatusCode(statusCode int) + + setRequestID(requestID string) + + setResponseHeaders(responseHeaders map[string][]string) +} + +// ISerializable defines interface with function: trans +type ISerializable interface { + trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) +} + +// DefaultSerializable defines default serializable struct +type DefaultSerializable struct { + params map[string]string + headers map[string][]string + data interface{} +} + +func (input DefaultSerializable) trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) { + return input.params, input.headers, input.data, nil +} + +var defaultSerializable = &DefaultSerializable{} + +func newSubResourceSerial(subResource SubResourceType) *DefaultSerializable { + return &DefaultSerializable{map[string]string{string(subResource): ""}, nil, nil} +} + +func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(subResource): ""} + data, err = ConvertRequestToIoReader(input) + return +} + +func (baseModel *BaseModel) setStatusCode(statusCode int) { + baseModel.StatusCode = statusCode +} + +func (baseModel *BaseModel) setRequestID(requestID string) { + baseModel.RequestId = requestID +} + +func (baseModel *BaseModel) setResponseHeaders(responseHeaders map[string][]string) { + baseModel.ResponseHeaders = responseHeaders +} + +func (input ListBucketsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if input.QueryLocation && !isObs { + setHeaders(headers, HEADER_LOCATION_AMZ, []string{"true"}, isObs) + } + return +} + +func (input CreateBucketInput) prepareGrantHeaders(headers map[string][]string, isObs bool) { + if grantReadID := input.GrantReadId; grantReadID != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{grantReadID}, isObs) + } + if grantWriteID := input.GrantWriteId; grantWriteID != "" { + setHeaders(headers, HEADER_GRANT_WRITE_OBS, []string{grantWriteID}, isObs) + } + if grantReadAcpID := input.GrantReadAcpId; grantReadAcpID != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{grantReadAcpID}, isObs) + } + if grantWriteAcpID := input.GrantWriteAcpId; grantWriteAcpID != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{grantWriteAcpID}, isObs) + } + if grantFullControlID := input.GrantFullControlId; grantFullControlID != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{grantFullControlID}, isObs) + } + if grantReadDeliveredID := input.GrantReadDeliveredId; grantReadDeliveredID != "" { + setHeaders(headers, HEADER_GRANT_READ_DELIVERED_OBS, []string{grantReadDeliveredID}, true) + } + if grantFullControlDeliveredID := input.GrantFullControlDeliveredId; grantFullControlDeliveredID != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS, []string{grantFullControlDeliveredID}, true) + } +} + +func (input CreateBucketInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeadersNext(headers, HEADER_STORAGE_CLASS_OBS, HEADER_STORAGE_CLASS, []string{storageClass}, isObs) + } + if epid := input.Epid; epid != "" { + setHeaders(headers, HEADER_EPID_HEADERS, []string{epid}, isObs) + } + if availableZone := input.AvailableZone; availableZone != "" { + setHeaders(headers, HEADER_AZ_REDUNDANCY, []string{availableZone}, isObs) + } + + input.prepareGrantHeaders(headers, isObs) + if input.IsFSFileInterface { + setHeaders(headers, headerFSFileInterface, []string{"Enabled"}, true) + } + + if location := strings.TrimSpace(input.Location); location != "" { + input.Location = location + + xml := make([]string, 0, 3) + xml = append(xml, "") + if isObs { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } else { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } + xml = append(xml, "") + + data = strings.Join(xml, "") + } + return +} + +func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + xml := make([]string, 0, 1) + if !isObs { + storageClass := "STANDARD" + if input.StorageClass == StorageClassWarm { + storageClass = string(storageClassStandardIA) + } else if input.StorageClass == StorageClassCold { + storageClass = string(storageClassGlacier) + } + params = map[string]string{string(SubResourceStoragePolicy): ""} + xml = append(xml, fmt.Sprintf("%s", storageClass)) + } else { + if input.StorageClass != StorageClassWarm && input.StorageClass != StorageClassCold { + input.StorageClass = StorageClassStandard + } + params = map[string]string{string(SubResourceStorageClass): ""} + xml = append(xml, fmt.Sprintf("%s", input.StorageClass)) + } + data = strings.Join(xml, "") + return +} + +func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.Prefix != "" { + params["prefix"] = input.Prefix + } + if input.Delimiter != "" { + params["delimiter"] = input.Delimiter + } + if input.MaxKeys > 0 { + params["max-keys"] = IntToString(input.MaxKeys) + } + headers = make(map[string][]string) + if origin := strings.TrimSpace(input.Origin); origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{origin} + } + if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader} + } + return +} + +func (input ListObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } + if input.Marker != "" { + params["marker"] = input.Marker + } + return +} + +func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } + params[string(SubResourceVersions)] = "" + if input.KeyMarker != "" { + params["key-marker"] = input.KeyMarker + } + if input.VersionIdMarker != "" { + params["version-id-marker"] = input.VersionIdMarker + } + return +} + +func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceUploads): ""} + if input.Prefix != "" { + params["prefix"] = input.Prefix + } + if input.Delimiter != "" { + params["delimiter"] = input.Delimiter + } + if input.MaxUploads > 0 { + params["max-uploads"] = IntToString(input.MaxUploads) + } + if input.KeyMarker != "" { + params["key-marker"] = input.KeyMarker + } + if input.UploadIdMarker != "" { + params["upload-id-marker"] = input.UploadIdMarker + } + return +} + +func (input SetBucketQuotaInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + return trans(SubResourceQuota, input) +} + +func (input SetBucketAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceAcl): ""} + headers = make(map[string][]string) + + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } else { + data, _ = convertBucketACLToXML(input.AccessControlPolicy, false, isObs) + } + return +} + +func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourcePolicy): ""} + data = strings.NewReader(input.Policy) + return +} + +func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceCors): ""} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketVersioningInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + return trans(SubResourceVersioning, input) +} + +func (input SetBucketWebsiteConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceWebsite): ""} + data, _ = ConvertWebsiteConfigurationToXml(input.BucketWebsiteConfiguration, false) + return +} + +func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if origin := strings.TrimSpace(input.Origin); origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{origin} + } + if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader} + } + return +} + +func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceLogging): ""} + data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false, isObs) + return +} + +func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceLifecycle): ""} + data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true, isObs) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceTagging): ""} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketNotificationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceNotification): ""} + data, _ = ConvertNotificationToXml(input.BucketNotification, false, isObs) + return +} + +func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceDelete): ""} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceAcl): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } else { + data, _ = ConvertAclToXml(input.AccessControlPolicy, false, isObs) + } + return +} + +func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceAcl): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input RestoreObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceRestore): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + if !isObs { + data, err = ConvertRequestToIoReader(input) + } else { + data = ConverntObsRestoreToXml(input) + } + return +} + +// GetEncryption gets the Encryption field value from SseKmsHeader +func (header SseKmsHeader) GetEncryption() string { + if header.Encryption != "" { + return header.Encryption + } + if !header.isObs { + return DEFAULT_SSE_KMS_ENCRYPTION + } + return DEFAULT_SSE_KMS_ENCRYPTION_OBS +} + +// GetKey gets the Key field value from SseKmsHeader +func (header SseKmsHeader) GetKey() string { + return header.Key +} + +// GetEncryption gets the Encryption field value from SseCHeader +func (header SseCHeader) GetEncryption() string { + if header.Encryption != "" { + return header.Encryption + } + return DEFAULT_SSE_C_ENCRYPTION +} + +// GetKey gets the Key field value from SseCHeader +func (header SseCHeader) GetKey() string { + return header.Key +} + +// GetKeyMD5 gets the KeyMD5 field value from SseCHeader +func (header SseCHeader) GetKeyMD5() string { + if header.KeyMD5 != "" { + return header.KeyMD5 + } + + if ret, err := Base64Decode(header.GetKey()); err == nil { + return Base64Md5(ret) + } + return "" +} + +func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bool, isObs bool) { + if sseHeader != nil { + if sseCHeader, ok := sseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } else if sseKmsHeader, ok := sseHeader.(SseKmsHeader); !sseCOnly && ok { + sseKmsHeader.isObs = isObs + setHeaders(headers, HEADER_SSEKMS_ENCRYPTION, []string{sseKmsHeader.GetEncryption()}, isObs) + if sseKmsHeader.GetKey() != "" { + setHeadersNext(headers, HEADER_SSEKMS_KEY_OBS, HEADER_SSEKMS_KEY_AMZ, []string{sseKmsHeader.GetKey()}, isObs) + } + } + } +} + +func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if input.Origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{input.Origin} + } + + if input.RequestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{input.RequestHeader} + } + setSseHeader(headers, input.SseHeader, true, isObs) + return +} + +func (input SetObjectMetadataInput) prepareContentHeaders(headers map[string][]string) { + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION_CAMEL] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING_CAMEL] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE_CAMEL] = []string{input.ContentLanguage} + } + + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } +} + +func (input SetObjectMetadataInput) prepareStorageClass(headers map[string][]string, isObs bool) { + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } +} + +func (input SetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + params = map[string]string{string(SubResourceMetadata): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(input.MetadataDirective)}, isObs) + } else { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(ReplaceNew)}, isObs) + } + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL_CAMEL] = []string{input.CacheControl} + } + input.prepareContentHeaders(headers) + if input.Expires != "" { + headers[HEADER_EXPIRES_CAMEL] = []string{input.Expires} + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + } + input.prepareStorageClass(headers, isObs) + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } + return +} + +func (input GetObjectInput) prepareResponseParams(params map[string]string) { + if input.ResponseCacheControl != "" { + params[PARAM_RESPONSE_CACHE_CONTROL] = input.ResponseCacheControl + } + if input.ResponseContentDisposition != "" { + params[PARAM_RESPONSE_CONTENT_DISPOSITION] = input.ResponseContentDisposition + } + if input.ResponseContentEncoding != "" { + params[PARAM_RESPONSE_CONTENT_ENCODING] = input.ResponseContentEncoding + } + if input.ResponseContentLanguage != "" { + params[PARAM_RESPONSE_CONTENT_LANGUAGE] = input.ResponseContentLanguage + } + if input.ResponseContentType != "" { + params[PARAM_RESPONSE_CONTENT_TYPE] = input.ResponseContentType + } + if input.ResponseExpires != "" { + params[PARAM_RESPONSE_EXPIRES] = input.ResponseExpires + } +} + +func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.GetObjectMetadataInput.trans(isObs) + if err != nil { + return + } + input.prepareResponseParams(params) + if input.ImageProcess != "" { + params[PARAM_IMAGE_PROCESS] = input.ImageProcess + } + if input.RangeStart >= 0 && input.RangeEnd > input.RangeStart { + headers[HEADER_RANGE] = []string{fmt.Sprintf("bytes=%d-%d", input.RangeStart, input.RangeEnd)} + } + + if input.IfMatch != "" { + headers[HEADER_IF_MATCH] = []string{input.IfMatch} + } + if input.IfNoneMatch != "" { + headers[HEADER_IF_NONE_MATCH] = []string{input.IfNoneMatch} + } + if !input.IfModifiedSince.IsZero() { + headers[HEADER_IF_MODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfModifiedSince)} + } + if !input.IfUnmodifiedSince.IsZero() { + headers[HEADER_IF_UNMODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfUnmodifiedSince)} + } + return +} + +func (input ObjectOperationInput) prepareGrantHeaders(headers map[string][]string) { + if GrantReadID := input.GrantReadId; GrantReadID != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{GrantReadID}, true) + } + if GrantReadAcpID := input.GrantReadAcpId; GrantReadAcpID != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{GrantReadAcpID}, true) + } + if GrantWriteAcpID := input.GrantWriteAcpId; GrantWriteAcpID != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{GrantWriteAcpID}, true) + } + if GrantFullControlID := input.GrantFullControlId; GrantFullControlID != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{GrantFullControlID}, true) + } +} + +func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + params = make(map[string]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + input.prepareGrantHeaders(headers) + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + + } + setSseHeader(headers, input.SseHeader, false, isObs) + if input.Expires != 0 { + setHeaders(headers, HEADER_EXPIRES, []string{Int64ToString(input.Expires)}, true) + } + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } + return +} + +func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } + + if input.ContentLength > 0 { + headers[HEADER_CONTENT_LENGTH_CAMEL] = []string{Int64ToString(input.ContentLength)} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } + + return +} + +func (input PutObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.PutObjectBasicInput.trans(isObs) + if err != nil { + return + } + if input.Body != nil { + data = input.Body + } + return +} + +func (input CopyObjectInput) prepareReplaceHeaders(headers map[string][]string) { + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL] = []string{input.CacheControl} + } + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE] = []string{input.ContentLanguage} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE] = []string{input.ContentType} + } + if input.Expires != "" { + headers[HEADER_EXPIRES] = []string{input.Expires} + } +} + +func (input CopyObjectInput) prepareCopySourceHeaders(headers map[string][]string, isObs bool) { + if input.CopySourceIfMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MATCH, []string{input.CopySourceIfMatch}, isObs) + } + if input.CopySourceIfNoneMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_NONE_MATCH, []string{input.CopySourceIfNoneMatch}, isObs) + } + if !input.CopySourceIfModifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}, isObs) + } + if !input.CopySourceIfUnmodifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}, isObs) + } +} + +func (input CopyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + + var copySource string + if input.CopySourceVersionId != "" { + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) + } else { + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) + } + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{directive}, isObs) + } + + if input.MetadataDirective == ReplaceMetadata { + input.prepareReplaceHeaders(headers) + } + + input.prepareCopySourceHeaders(headers, isObs) + if input.SourceSseHeader != nil { + if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } + } + if input.SuccessActionRedirect != "" { + headers[HEADER_SUCCESS_ACTION_REDIRECT] = []string{input.SuccessActionRedirect} + } + return +} + +func (input AbortMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId} + return +} + +func (input InitiateMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } + params[string(SubResourceUploads)] = "" + return +} + +func (input UploadPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} + headers = make(map[string][]string) + setSseHeader(headers, input.SseHeader, true, isObs) + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } + if input.Body != nil { + data = input.Body + } + return +} + +func (input CompleteMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId} + data, _ = ConvertCompleteMultipartUploadInputToXml(input, false) + return +} + +func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId} + if input.MaxParts > 0 { + params["max-parts"] = IntToString(input.MaxParts) + } + if input.PartNumberMarker > 0 { + params["part-number-marker"] = IntToString(input.PartNumberMarker) + } + return +} + +func (input CopyPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} + headers = make(map[string][]string, 1) + var copySource string + if input.CopySourceVersionId != "" { + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) + } else { + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) + } + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) + if input.CopySourceRangeStart >= 0 && input.CopySourceRangeEnd > input.CopySourceRangeStart { + setHeaders(headers, HEADER_COPY_SOURCE_RANGE, []string{fmt.Sprintf("bytes=%d-%d", input.CopySourceRangeStart, input.CopySourceRangeEnd)}, isObs) + } + + setSseHeader(headers, input.SseHeader, true, isObs) + if input.SourceSseHeader != nil { + if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } + + } + return +} + +func (input HeadObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input SetBucketRequestPaymentInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + return trans(SubResourceRequestPayment, input) +} + +type partSlice []Part + +func (parts partSlice) Len() int { + return len(parts) +} + +func (parts partSlice) Less(i, j int) bool { + return parts[i].PartNumber < parts[j].PartNumber +} + +func (parts partSlice) Swap(i, j int) { + parts[i], parts[j] = parts[j], parts[i] +} + +type readerWrapper struct { + reader io.Reader + mark int64 + totalCount int64 + readedCount int64 +} + +func (rw *readerWrapper) seek(offset int64, whence int) (int64, error) { + if r, ok := rw.reader.(*strings.Reader); ok { + return r.Seek(offset, whence) + } else if r, ok := rw.reader.(*bytes.Reader); ok { + return r.Seek(offset, whence) + } else if r, ok := rw.reader.(*os.File); ok { + return r.Seek(offset, whence) + } + return offset, nil +} + +func (rw *readerWrapper) Read(p []byte) (n int, err error) { + if rw.totalCount == 0 { + return 0, io.EOF + } + if rw.totalCount > 0 { + n, err = rw.reader.Read(p) + readedOnce := int64(n) + remainCount := rw.totalCount - rw.readedCount + if remainCount > readedOnce { + rw.readedCount += readedOnce + return n, err + } + rw.readedCount += remainCount + return int(remainCount), io.EOF + } + return rw.reader.Read(p) +} + +type fileReaderWrapper struct { + readerWrapper + filePath string +} + +func (input SetBucketFetchPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + contentType, _ := mimeTypes["json"] + headers = make(map[string][]string, 2) + headers[HEADER_CONTENT_TYPE] = []string{contentType} + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + data, err = convertFetchPolicyToJSON(input) + return +} + +func (input GetBucketFetchPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string, 1) + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + return +} + +func (input DeleteBucketFetchPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string, 1) + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + return +} + +func (input SetBucketFetchJobInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + contentType, _ := mimeTypes["json"] + headers = make(map[string][]string, 2) + headers[HEADER_CONTENT_TYPE] = []string{contentType} + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + data, err = convertFetchJobToJSON(input) + return +} + +func (input GetBucketFetchJobInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string, 1) + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + return +} diff --git a/cve-vulner-manager/obs/transfer.go b/cve-vulner-manager/obs/transfer.go new file mode 100644 index 0000000000000000000000000000000000000000..a2d3664e9afc1ca6f49e7f363f05a2692af6599b --- /dev/null +++ b/cve-vulner-manager/obs/transfer.go @@ -0,0 +1,872 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "bufio" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sync" + "sync/atomic" + "syscall" +) + +var errAbort = errors.New("AbortError") + +// FileStatus defines the upload file properties +type FileStatus struct { + XMLName xml.Name `xml:"FileInfo"` + LastModified int64 `xml:"LastModified"` + Size int64 `xml:"Size"` +} + +// UploadPartInfo defines the upload part properties +type UploadPartInfo struct { + XMLName xml.Name `xml:"UploadPart"` + PartNumber int `xml:"PartNumber"` + Etag string `xml:"Etag"` + PartSize int64 `xml:"PartSize"` + Offset int64 `xml:"Offset"` + IsCompleted bool `xml:"IsCompleted"` +} + +// UploadCheckpoint defines the upload checkpoint file properties +type UploadCheckpoint struct { + XMLName xml.Name `xml:"UploadFileCheckpoint"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId,omitempty"` + UploadFile string `xml:"FileUrl"` + FileInfo FileStatus `xml:"FileInfo"` + UploadParts []UploadPartInfo `xml:"UploadParts>UploadPart"` +} + +func (ufc *UploadCheckpoint) isValid(bucket, key, uploadFile string, fileStat os.FileInfo) bool { + if ufc.Bucket != bucket || ufc.Key != key || ufc.UploadFile != uploadFile { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the bucketName or objectKey or uploadFile was changed. clear the record.") + return false + } + + if ufc.FileInfo.Size != fileStat.Size() || ufc.FileInfo.LastModified != fileStat.ModTime().Unix() { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the uploadFile was changed. clear the record.") + return false + } + + if ufc.UploadId == "" { + doLog(LEVEL_INFO, "UploadId is invalid. clear the record.") + return false + } + + return true +} + +type uploadPartTask struct { + UploadPartInput + obsClient *ObsClient + abort *int32 + extensions []extensionOptions + enableCheckpoint bool +} + +func (task *uploadPartTask) Run() interface{} { + if atomic.LoadInt32(task.abort) == 1 { + return errAbort + } + + input := &UploadPartInput{} + input.Bucket = task.Bucket + input.Key = task.Key + input.PartNumber = task.PartNumber + input.UploadId = task.UploadId + input.SseHeader = task.SseHeader + input.SourceFile = task.SourceFile + input.Offset = task.Offset + input.PartSize = task.PartSize + extensions := task.extensions + + var output *UploadPartOutput + var err error + if extensions != nil { + output, err = task.obsClient.UploadPart(input, extensions...) + } else { + output, err = task.obsClient.UploadPart(input) + } + + if err == nil { + if output.ETag == "" { + doLog(LEVEL_WARN, "Get invalid etag value after uploading part [%d].", task.PartNumber) + if !task.enableCheckpoint { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.PartNumber) + } + return fmt.Errorf("get invalid etag value after uploading part [%d]", task.PartNumber) + } + return output + } else if obsError, ok := err.(ObsError); ok && obsError.StatusCode >= 400 && obsError.StatusCode < 500 { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.PartNumber) + } + return err +} + +func loadCheckpointFile(checkpointFile string, result interface{}) error { + ret, err := ioutil.ReadFile(checkpointFile) + if err != nil { + return err + } + if len(ret) == 0 { + return nil + } + return xml.Unmarshal(ret, result) +} + +func updateCheckpointFile(fc interface{}, checkpointFilePath string) error { + result, err := xml.Marshal(fc) + if err != nil { + return err + } + err = ioutil.WriteFile(checkpointFilePath, result, 0666) + return err +} + +func getCheckpointFile(ufc *UploadCheckpoint, uploadFileStat os.FileInfo, input *UploadFileInput, obsClient *ObsClient, extensions []extensionOptions) (needCheckpoint bool, err error) { + checkpointFilePath := input.CheckpointFile + checkpointFileStat, err := os.Stat(checkpointFilePath) + if err != nil { + doLog(LEVEL_DEBUG, fmt.Sprintf("Stat checkpoint file failed with error: [%v].", err)) + return true, nil + } + if checkpointFileStat.IsDir() { + doLog(LEVEL_ERROR, "Checkpoint file can not be a folder.") + return false, errors.New("checkpoint file can not be a folder") + } + err = loadCheckpointFile(checkpointFilePath, ufc) + if err != nil { + doLog(LEVEL_WARN, fmt.Sprintf("Load checkpoint file failed with error: [%v].", err)) + return true, nil + } else if !ufc.isValid(input.Bucket, input.Key, input.UploadFile, uploadFileStat) { + if ufc.Bucket != "" && ufc.Key != "" && ufc.UploadId != "" { + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort upload task [%s].", ufc.UploadId) + } + } + _err := os.Remove(checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, fmt.Sprintf("Failed to remove checkpoint file with error: [%v].", _err)) + } + } else { + return false, nil + } + + return true, nil +} + +func prepareUpload(ufc *UploadCheckpoint, uploadFileStat os.FileInfo, input *UploadFileInput, obsClient *ObsClient, extensions []extensionOptions) error { + initiateInput := &InitiateMultipartUploadInput{} + initiateInput.ObjectOperationInput = input.ObjectOperationInput + initiateInput.ContentType = input.ContentType + var output *InitiateMultipartUploadOutput + var err error + if extensions != nil { + output, err = obsClient.InitiateMultipartUpload(initiateInput, extensions...) + } else { + output, err = obsClient.InitiateMultipartUpload(initiateInput) + } + if err != nil { + return err + } + + ufc.Bucket = input.Bucket + ufc.Key = input.Key + ufc.UploadFile = input.UploadFile + ufc.FileInfo = FileStatus{} + ufc.FileInfo.Size = uploadFileStat.Size() + ufc.FileInfo.LastModified = uploadFileStat.ModTime().Unix() + ufc.UploadId = output.UploadId + + err = sliceFile(input.PartSize, ufc) + return err +} + +func sliceFile(partSize int64, ufc *UploadCheckpoint) error { + fileSize := ufc.FileInfo.Size + cnt := fileSize / partSize + if cnt >= 10000 { + partSize = fileSize / 10000 + if fileSize%10000 != 0 { + partSize++ + } + cnt = fileSize / partSize + } + if fileSize%partSize != 0 { + cnt++ + } + + if partSize > MAX_PART_SIZE { + doLog(LEVEL_ERROR, "The source upload file is too large") + return fmt.Errorf("The source upload file is too large") + } + + if cnt == 0 { + uploadPart := UploadPartInfo{} + uploadPart.PartNumber = 1 + ufc.UploadParts = []UploadPartInfo{uploadPart} + } else { + uploadParts := make([]UploadPartInfo, 0, cnt) + var i int64 + for i = 0; i < cnt; i++ { + uploadPart := UploadPartInfo{} + uploadPart.PartNumber = int(i) + 1 + uploadPart.PartSize = partSize + uploadPart.Offset = i * partSize + uploadParts = append(uploadParts, uploadPart) + } + if value := fileSize % partSize; value != 0 { + uploadParts[cnt-1].PartSize = value + } + ufc.UploadParts = uploadParts + } + return nil +} + +func abortTask(bucket, key, uploadID string, obsClient *ObsClient, extensions []extensionOptions) error { + input := &AbortMultipartUploadInput{} + input.Bucket = bucket + input.Key = key + input.UploadId = uploadID + if extensions != nil { + _, err := obsClient.AbortMultipartUpload(input, extensions...) + return err + } + _, err := obsClient.AbortMultipartUpload(input) + return err +} + +func handleUploadFileResult(uploadPartError error, ufc *UploadCheckpoint, enableCheckpoint bool, obsClient *ObsClient, extensions []extensionOptions) error { + if uploadPartError != nil { + if enableCheckpoint { + return uploadPartError + } + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort task [%s].", ufc.UploadId) + } + return uploadPartError + } + return nil +} + +func completeParts(ufc *UploadCheckpoint, enableCheckpoint bool, checkpointFilePath string, obsClient *ObsClient, extensions []extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + completeInput := &CompleteMultipartUploadInput{} + completeInput.Bucket = ufc.Bucket + completeInput.Key = ufc.Key + completeInput.UploadId = ufc.UploadId + parts := make([]Part, 0, len(ufc.UploadParts)) + for _, uploadPart := range ufc.UploadParts { + part := Part{} + part.PartNumber = uploadPart.PartNumber + part.ETag = uploadPart.Etag + parts = append(parts, part) + } + completeInput.Parts = parts + var completeOutput *CompleteMultipartUploadOutput + if extensions != nil { + completeOutput, err = obsClient.CompleteMultipartUpload(completeInput, extensions...) + } else { + completeOutput, err = obsClient.CompleteMultipartUpload(completeInput) + } + + if err == nil { + if enableCheckpoint { + _err := os.Remove(checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, "Upload file successfully, but remove checkpoint file failed with error [%v].", _err) + } + } + return completeOutput, err + } + if !enableCheckpoint { + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort task [%s].", ufc.UploadId) + } + } + return completeOutput, err +} + +func (obsClient ObsClient) resumeUpload(input *UploadFileInput, extensions []extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + uploadFileStat, err := os.Stat(input.UploadFile) + if err != nil { + doLog(LEVEL_ERROR, fmt.Sprintf("Failed to stat uploadFile with error: [%v].", err)) + return nil, err + } + if uploadFileStat.IsDir() { + doLog(LEVEL_ERROR, "UploadFile can not be a folder.") + return nil, errors.New("uploadFile can not be a folder") + } + + ufc := &UploadCheckpoint{} + + var needCheckpoint = true + var checkpointFilePath = input.CheckpointFile + var enableCheckpoint = input.EnableCheckpoint + if enableCheckpoint { + needCheckpoint, err = getCheckpointFile(ufc, uploadFileStat, input, &obsClient, extensions) + if err != nil { + return nil, err + } + } + if needCheckpoint { + err = prepareUpload(ufc, uploadFileStat, input, &obsClient, extensions) + if err != nil { + return nil, err + } + + if enableCheckpoint { + err = updateCheckpointFile(ufc, checkpointFilePath) + if err != nil { + doLog(LEVEL_ERROR, "Failed to update checkpoint file with error [%v].", err) + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, &obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort task [%s].", ufc.UploadId) + } + return nil, err + } + } + } + + uploadPartError := obsClient.uploadPartConcurrent(ufc, checkpointFilePath, input, extensions) + err = handleUploadFileResult(uploadPartError, ufc, enableCheckpoint, &obsClient, extensions) + if err != nil { + return nil, err + } + + completeOutput, err := completeParts(ufc, enableCheckpoint, checkpointFilePath, &obsClient, extensions) + + return completeOutput, err +} + +func handleUploadTaskResult(result interface{}, ufc *UploadCheckpoint, partNum int, enableCheckpoint bool, checkpointFilePath string, lock *sync.Mutex) (err error) { + if uploadPartOutput, ok := result.(*UploadPartOutput); ok { + lock.Lock() + defer lock.Unlock() + ufc.UploadParts[partNum-1].Etag = uploadPartOutput.ETag + ufc.UploadParts[partNum-1].IsCompleted = true + if enableCheckpoint { + _err := updateCheckpointFile(ufc, checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, "Failed to update checkpoint file with error [%v].", _err) + } + } + } else if result != errAbort { + if _err, ok := result.(error); ok { + err = _err + } + } + return +} + +func (obsClient ObsClient) uploadPartConcurrent(ufc *UploadCheckpoint, checkpointFilePath string, input *UploadFileInput, extensions []extensionOptions) error { + pool := NewRoutinePool(input.TaskNum, MAX_PART_NUM) + var uploadPartError atomic.Value + var errFlag int32 + var abort int32 + lock := new(sync.Mutex) + for _, uploadPart := range ufc.UploadParts { + if atomic.LoadInt32(&abort) == 1 { + break + } + if uploadPart.IsCompleted { + continue + } + task := uploadPartTask{ + UploadPartInput: UploadPartInput{ + Bucket: ufc.Bucket, + Key: ufc.Key, + PartNumber: uploadPart.PartNumber, + UploadId: ufc.UploadId, + SseHeader: input.SseHeader, + SourceFile: input.UploadFile, + Offset: uploadPart.Offset, + PartSize: uploadPart.PartSize, + }, + obsClient: &obsClient, + abort: &abort, + extensions: extensions, + enableCheckpoint: input.EnableCheckpoint, + } + pool.ExecuteFunc(func() interface{} { + result := task.Run() + err := handleUploadTaskResult(result, ufc, task.PartNumber, input.EnableCheckpoint, input.CheckpointFile, lock) + if err != nil && atomic.CompareAndSwapInt32(&errFlag, 0, 1) { + uploadPartError.Store(err) + } + return nil + }) + } + pool.ShutDown() + if err, ok := uploadPartError.Load().(error); ok { + return err + } + return nil +} + +// ObjectInfo defines download object info +type ObjectInfo struct { + XMLName xml.Name `xml:"ObjectInfo"` + LastModified int64 `xml:"LastModified"` + Size int64 `xml:"Size"` + ETag string `xml:"ETag"` +} + +// TempFileInfo defines temp download file properties +type TempFileInfo struct { + XMLName xml.Name `xml:"TempFileInfo"` + TempFileUrl string `xml:"TempFileUrl"` + Size int64 `xml:"Size"` +} + +// DownloadPartInfo defines download part properties +type DownloadPartInfo struct { + XMLName xml.Name `xml:"DownloadPart"` + PartNumber int64 `xml:"PartNumber"` + RangeEnd int64 `xml:"RangeEnd"` + Offset int64 `xml:"Offset"` + IsCompleted bool `xml:"IsCompleted"` +} + +// DownloadCheckpoint defines download checkpoint file properties +type DownloadCheckpoint struct { + XMLName xml.Name `xml:"DownloadFileCheckpoint"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` + DownloadFile string `xml:"FileUrl"` + ObjectInfo ObjectInfo `xml:"ObjectInfo"` + TempFileInfo TempFileInfo `xml:"TempFileInfo"` + DownloadParts []DownloadPartInfo `xml:"DownloadParts>DownloadPart"` +} + +func (dfc *DownloadCheckpoint) isValid(input *DownloadFileInput, output *GetObjectMetadataOutput) bool { + if dfc.Bucket != input.Bucket || dfc.Key != input.Key || dfc.VersionId != input.VersionId || dfc.DownloadFile != input.DownloadFile { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the bucketName or objectKey or downloadFile was changed. clear the record.") + return false + } + if dfc.ObjectInfo.LastModified != output.LastModified.Unix() || dfc.ObjectInfo.ETag != output.ETag || dfc.ObjectInfo.Size != output.ContentLength { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the object info was changed. clear the record.") + return false + } + if dfc.TempFileInfo.Size != output.ContentLength { + doLog(LEVEL_INFO, "Checkpoint file is invalid, size was changed. clear the record.") + return false + } + stat, err := os.Stat(dfc.TempFileInfo.TempFileUrl) + if err != nil || stat.Size() != dfc.ObjectInfo.Size { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the temp download file was changed. clear the record.") + return false + } + + return true +} + +type downloadPartTask struct { + GetObjectInput + obsClient *ObsClient + extensions []extensionOptions + abort *int32 + partNumber int64 + tempFileURL string + enableCheckpoint bool +} + +func (task *downloadPartTask) Run() interface{} { + if atomic.LoadInt32(task.abort) == 1 { + return errAbort + } + getObjectInput := &GetObjectInput{} + getObjectInput.GetObjectMetadataInput = task.GetObjectMetadataInput + getObjectInput.IfMatch = task.IfMatch + getObjectInput.IfNoneMatch = task.IfNoneMatch + getObjectInput.IfModifiedSince = task.IfModifiedSince + getObjectInput.IfUnmodifiedSince = task.IfUnmodifiedSince + getObjectInput.RangeStart = task.RangeStart + getObjectInput.RangeEnd = task.RangeEnd + + var output *GetObjectOutput + var err error + if task.extensions != nil { + output, err = task.obsClient.GetObject(getObjectInput, task.extensions...) + } else { + output, err = task.obsClient.GetObject(getObjectInput) + } + + if err == nil { + defer func() { + errMsg := output.Body.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close response body.") + } + }() + _err := updateDownloadFile(task.tempFileURL, task.RangeStart, output) + if _err != nil { + if !task.enableCheckpoint { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.partNumber) + } + return _err + } + return output + } else if obsError, ok := err.(ObsError); ok && obsError.StatusCode >= 400 && obsError.StatusCode < 500 { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.partNumber) + } + return err +} + +func getObjectInfo(input *DownloadFileInput, obsClient *ObsClient, extensions []extensionOptions) (getObjectmetaOutput *GetObjectMetadataOutput, err error) { + if extensions != nil { + getObjectmetaOutput, err = obsClient.GetObjectMetadata(&input.GetObjectMetadataInput, extensions...) + } else { + getObjectmetaOutput, err = obsClient.GetObjectMetadata(&input.GetObjectMetadataInput) + } + + return +} + +func getDownloadCheckpointFile(dfc *DownloadCheckpoint, input *DownloadFileInput, output *GetObjectMetadataOutput) (needCheckpoint bool, err error) { + checkpointFilePath := input.CheckpointFile + checkpointFileStat, err := os.Stat(checkpointFilePath) + if err != nil { + doLog(LEVEL_DEBUG, fmt.Sprintf("Stat checkpoint file failed with error: [%v].", err)) + return true, nil + } + if checkpointFileStat.IsDir() { + doLog(LEVEL_ERROR, "Checkpoint file can not be a folder.") + return false, errors.New("checkpoint file can not be a folder") + } + err = loadCheckpointFile(checkpointFilePath, dfc) + if err != nil { + doLog(LEVEL_WARN, fmt.Sprintf("Load checkpoint file failed with error: [%v].", err)) + return true, nil + } else if !dfc.isValid(input, output) { + if dfc.TempFileInfo.TempFileUrl != "" { + _err := os.Remove(dfc.TempFileInfo.TempFileUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to remove temp download file with error [%v].", _err) + } + } + _err := os.Remove(checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, "Failed to remove checkpoint file with error [%v].", _err) + } + } else { + return false, nil + } + + return true, nil +} + +func sliceObject(objectSize, partSize int64, dfc *DownloadCheckpoint) { + cnt := objectSize / partSize + if objectSize%partSize > 0 { + cnt++ + } + + if cnt == 0 { + downloadPart := DownloadPartInfo{} + downloadPart.PartNumber = 1 + dfc.DownloadParts = []DownloadPartInfo{downloadPart} + } else { + downloadParts := make([]DownloadPartInfo, 0, cnt) + var i int64 + for i = 0; i < cnt; i++ { + downloadPart := DownloadPartInfo{} + downloadPart.PartNumber = i + 1 + downloadPart.Offset = i * partSize + downloadPart.RangeEnd = (i+1)*partSize - 1 + downloadParts = append(downloadParts, downloadPart) + } + dfc.DownloadParts = downloadParts + if value := objectSize % partSize; value > 0 { + dfc.DownloadParts[cnt-1].RangeEnd = dfc.ObjectInfo.Size - 1 + } + } +} + +func createFile(tempFileURL string, fileSize int64) error { + fd, err := syscall.Open(tempFileURL, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + doLog(LEVEL_WARN, "Failed to open temp download file [%s].", tempFileURL) + return err + } + defer func() { + errMsg := syscall.Close(fd) + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with error [%v].", errMsg) + } + }() + err = syscall.Ftruncate(fd, fileSize) + if err != nil { + doLog(LEVEL_WARN, "Failed to create file with error [%v].", err) + } + return err +} + +func prepareTempFile(tempFileURL string, fileSize int64) error { + parentDir := filepath.Dir(tempFileURL) + stat, err := os.Stat(parentDir) + if err != nil { + doLog(LEVEL_DEBUG, "Failed to stat path with error [%v].", err) + _err := os.MkdirAll(parentDir, os.ModePerm) + if _err != nil { + doLog(LEVEL_ERROR, "Failed to make dir with error [%v].", _err) + return _err + } + } else if !stat.IsDir() { + doLog(LEVEL_ERROR, "Cannot create folder [%s] due to a same file exists.", parentDir) + return fmt.Errorf("cannot create folder [%s] due to a same file exists", parentDir) + } + + err = createFile(tempFileURL, fileSize) + if err == nil { + return nil + } + fd, err := os.OpenFile(tempFileURL, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + doLog(LEVEL_ERROR, "Failed to open temp download file [%s].", tempFileURL) + return err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with error [%v].", errMsg) + } + }() + if fileSize > 0 { + _, err = fd.WriteAt([]byte("a"), fileSize-1) + if err != nil { + doLog(LEVEL_ERROR, "Failed to create temp download file with error [%v].", err) + return err + } + } + + return nil +} + +func handleDownloadFileResult(tempFileURL string, enableCheckpoint bool, downloadFileError error) error { + if downloadFileError != nil { + if !enableCheckpoint { + _err := os.Remove(tempFileURL) + if _err != nil { + doLog(LEVEL_WARN, "Failed to remove temp download file with error [%v].", _err) + } + } + return downloadFileError + } + return nil +} + +func (obsClient ObsClient) resumeDownload(input *DownloadFileInput, extensions []extensionOptions) (output *GetObjectMetadataOutput, err error) { + getObjectmetaOutput, err := getObjectInfo(input, &obsClient, extensions) + if err != nil { + return nil, err + } + + objectSize := getObjectmetaOutput.ContentLength + partSize := input.PartSize + dfc := &DownloadCheckpoint{} + + var needCheckpoint = true + var checkpointFilePath = input.CheckpointFile + var enableCheckpoint = input.EnableCheckpoint + if enableCheckpoint { + needCheckpoint, err = getDownloadCheckpointFile(dfc, input, getObjectmetaOutput) + if err != nil { + return nil, err + } + } + + if needCheckpoint { + dfc.Bucket = input.Bucket + dfc.Key = input.Key + dfc.VersionId = input.VersionId + dfc.DownloadFile = input.DownloadFile + dfc.ObjectInfo = ObjectInfo{} + dfc.ObjectInfo.LastModified = getObjectmetaOutput.LastModified.Unix() + dfc.ObjectInfo.Size = getObjectmetaOutput.ContentLength + dfc.ObjectInfo.ETag = getObjectmetaOutput.ETag + dfc.TempFileInfo = TempFileInfo{} + dfc.TempFileInfo.TempFileUrl = input.DownloadFile + ".tmp" + dfc.TempFileInfo.Size = getObjectmetaOutput.ContentLength + + sliceObject(objectSize, partSize, dfc) + _err := prepareTempFile(dfc.TempFileInfo.TempFileUrl, dfc.TempFileInfo.Size) + if _err != nil { + return nil, _err + } + + if enableCheckpoint { + _err := updateCheckpointFile(dfc, checkpointFilePath) + if _err != nil { + doLog(LEVEL_ERROR, "Failed to update checkpoint file with error [%v].", _err) + _errMsg := os.Remove(dfc.TempFileInfo.TempFileUrl) + if _errMsg != nil { + doLog(LEVEL_WARN, "Failed to remove temp download file with error [%v].", _errMsg) + } + return nil, _err + } + } + } + + downloadFileError := obsClient.downloadFileConcurrent(input, dfc, extensions) + err = handleDownloadFileResult(dfc.TempFileInfo.TempFileUrl, enableCheckpoint, downloadFileError) + if err != nil { + return nil, err + } + + err = os.Rename(dfc.TempFileInfo.TempFileUrl, input.DownloadFile) + if err != nil { + doLog(LEVEL_ERROR, "Failed to rename temp download file [%s] to download file [%s] with error [%v].", dfc.TempFileInfo.TempFileUrl, input.DownloadFile, err) + return nil, err + } + if enableCheckpoint { + err = os.Remove(checkpointFilePath) + if err != nil { + doLog(LEVEL_WARN, "Download file successfully, but remove checkpoint file failed with error [%v].", err) + } + } + + return getObjectmetaOutput, nil +} + +func updateDownloadFile(filePath string, rangeStart int64, output *GetObjectOutput) error { + fd, err := os.OpenFile(filePath, os.O_WRONLY, 0666) + if err != nil { + doLog(LEVEL_ERROR, "Failed to open file [%s].", filePath) + return err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with error [%v].", errMsg) + } + }() + _, err = fd.Seek(rangeStart, 0) + if err != nil { + doLog(LEVEL_ERROR, "Failed to seek file with error [%v].", err) + return err + } + fileWriter := bufio.NewWriterSize(fd, 65536) + part := make([]byte, 8192) + var readErr error + var readCount int + for { + readCount, readErr = output.Body.Read(part) + if readCount > 0 { + wcnt, werr := fileWriter.Write(part[0:readCount]) + if werr != nil { + doLog(LEVEL_ERROR, "Failed to write to file with error [%v].", werr) + return werr + } + if wcnt != readCount { + doLog(LEVEL_ERROR, "Failed to write to file [%s], expect: [%d], actual: [%d]", filePath, readCount, wcnt) + return fmt.Errorf("Failed to write to file [%s], expect: [%d], actual: [%d]", filePath, readCount, wcnt) + } + } + if readErr != nil { + if readErr != io.EOF { + doLog(LEVEL_ERROR, "Failed to read response body with error [%v].", readErr) + return readErr + } + break + } + } + err = fileWriter.Flush() + if err != nil { + doLog(LEVEL_ERROR, "Failed to flush file with error [%v].", err) + return err + } + return nil +} + +func handleDownloadTaskResult(result interface{}, dfc *DownloadCheckpoint, partNum int64, enableCheckpoint bool, checkpointFile string, lock *sync.Mutex) (err error) { + if _, ok := result.(*GetObjectOutput); ok { + lock.Lock() + defer lock.Unlock() + dfc.DownloadParts[partNum-1].IsCompleted = true + if enableCheckpoint { + _err := updateCheckpointFile(dfc, checkpointFile) + if _err != nil { + doLog(LEVEL_WARN, "Failed to update checkpoint file with error [%v].", _err) + } + } + } else if result != errAbort { + if _err, ok := result.(error); ok { + err = _err + } + } + return +} + +func (obsClient ObsClient) downloadFileConcurrent(input *DownloadFileInput, dfc *DownloadCheckpoint, extensions []extensionOptions) error { + pool := NewRoutinePool(input.TaskNum, MAX_PART_NUM) + var downloadPartError atomic.Value + var errFlag int32 + var abort int32 + lock := new(sync.Mutex) + for _, downloadPart := range dfc.DownloadParts { + if atomic.LoadInt32(&abort) == 1 { + break + } + if downloadPart.IsCompleted { + continue + } + task := downloadPartTask{ + GetObjectInput: GetObjectInput{ + GetObjectMetadataInput: input.GetObjectMetadataInput, + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + IfUnmodifiedSince: input.IfUnmodifiedSince, + IfModifiedSince: input.IfModifiedSince, + RangeStart: downloadPart.Offset, + RangeEnd: downloadPart.RangeEnd, + }, + obsClient: &obsClient, + extensions: extensions, + abort: &abort, + partNumber: downloadPart.PartNumber, + tempFileURL: dfc.TempFileInfo.TempFileUrl, + enableCheckpoint: input.EnableCheckpoint, + } + pool.ExecuteFunc(func() interface{} { + result := task.Run() + err := handleDownloadTaskResult(result, dfc, task.partNumber, input.EnableCheckpoint, input.CheckpointFile, lock) + if err != nil && atomic.CompareAndSwapInt32(&errFlag, 0, 1) { + downloadPartError.Store(err) + } + return nil + }) + } + pool.ShutDown() + if err, ok := downloadPartError.Load().(error); ok { + return err + } + + return nil +} diff --git a/cve-vulner-manager/obs/util.go b/cve-vulner-manager/obs/util.go new file mode 100644 index 0000000000000000000000000000000000000000..3c728a05ce6f028f0776834d6915d0cddb8c5d5f --- /dev/null +++ b/cve-vulner-manager/obs/util.go @@ -0,0 +1,535 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/xml" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +var regex = regexp.MustCompile("^[\u4e00-\u9fa5]$") +var ipRegex = regexp.MustCompile("^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$") +var v4AuthRegex = regexp.MustCompile("Credential=(.+?),SignedHeaders=(.+?),Signature=.+") +var regionRegex = regexp.MustCompile(".+/\\d+/(.+?)/.+") + +// StringContains replaces subStr in src with subTranscoding and returns the new string +func StringContains(src string, subStr string, subTranscoding string) string { + return strings.Replace(src, subStr, subTranscoding, -1) +} + +// XmlTranscoding replaces special characters with their escaped form +func XmlTranscoding(src string) string { + srcTmp := StringContains(src, "&", "&") + srcTmp = StringContains(srcTmp, "<", "<") + srcTmp = StringContains(srcTmp, ">", ">") + srcTmp = StringContains(srcTmp, "'", "'") + srcTmp = StringContains(srcTmp, "\"", """) + return srcTmp +} + +// StringToInt converts string value to int value with default value +func StringToInt(value string, def int) int { + ret, err := strconv.Atoi(value) + if err != nil { + ret = def + } + return ret +} + +// StringToInt64 converts string value to int64 value with default value +func StringToInt64(value string, def int64) int64 { + ret, err := strconv.ParseInt(value, 10, 64) + if err != nil { + ret = def + } + return ret +} + +// IntToString converts int value to string value +func IntToString(value int) string { + return strconv.Itoa(value) +} + +// Int64ToString converts int64 value to string value +func Int64ToString(value int64) string { + return strconv.FormatInt(value, 10) +} + +// GetCurrentTimestamp gets unix time in milliseconds +func GetCurrentTimestamp() int64 { + return time.Now().UnixNano() / 1000000 +} + +// FormatUtcNow gets a textual representation of the UTC format time value +func FormatUtcNow(format string) string { + return time.Now().UTC().Format(format) +} + +// FormatUtcToRfc1123 gets a textual representation of the RFC1123 format time value +func FormatUtcToRfc1123(t time.Time) string { + ret := t.UTC().Format(time.RFC1123) + return ret[:strings.LastIndex(ret, "UTC")] + "GMT" +} + +// Md5 gets the md5 value of input +func Md5(value []byte) []byte { + m := md5.New() + _, err := m.Write(value) + if err != nil { + doLog(LEVEL_WARN, "MD5 failed to write") + } + return m.Sum(nil) +} + +// HmacSha1 gets hmac sha1 value of input +func HmacSha1(key, value []byte) []byte { + mac := hmac.New(sha1.New, key) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha1 failed to write") + } + return mac.Sum(nil) +} + +// HmacSha256 get hmac sha256 value if input +func HmacSha256(key, value []byte) []byte { + mac := hmac.New(sha256.New, key) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha256 failed to write") + } + return mac.Sum(nil) +} + +// Base64Encode wrapper of base64.StdEncoding.EncodeToString +func Base64Encode(value []byte) string { + return base64.StdEncoding.EncodeToString(value) +} + +// Base64Decode wrapper of base64.StdEncoding.DecodeString +func Base64Decode(value string) ([]byte, error) { + return base64.StdEncoding.DecodeString(value) +} + +// HexMd5 returns the md5 value of input in hexadecimal format +func HexMd5(value []byte) string { + return Hex(Md5(value)) +} + +// Base64Md5 returns the md5 value of input with Base64Encode +func Base64Md5(value []byte) string { + return Base64Encode(Md5(value)) +} + +// Sha256Hash returns sha256 checksum +func Sha256Hash(value []byte) []byte { + hash := sha256.New() + _, err := hash.Write(value) + if err != nil { + doLog(LEVEL_WARN, "Sha256Hash failed to write") + } + return hash.Sum(nil) +} + +// ParseXml wrapper of xml.Unmarshal +func ParseXml(value []byte, result interface{}) error { + if len(value) == 0 { + return nil + } + return xml.Unmarshal(value, result) +} + +// parseJSON wrapper of json.Unmarshal +func parseJSON(value []byte, result interface{}) error { + if len(value) == 0 { + return nil + } + return json.Unmarshal(value, result) +} + +// TransToXml wrapper of xml.Marshal +func TransToXml(value interface{}) ([]byte, error) { + if value == nil { + return []byte{}, nil + } + return xml.Marshal(value) +} + +// Hex wrapper of hex.EncodeToString +func Hex(value []byte) string { + return hex.EncodeToString(value) +} + +// HexSha256 returns the Sha256Hash value of input in hexadecimal format +func HexSha256(value []byte) string { + return Hex(Sha256Hash(value)) +} + +// UrlDecode wrapper of url.QueryUnescape +func UrlDecode(value string) (string, error) { + ret, err := url.QueryUnescape(value) + if err == nil { + return ret, nil + } + return "", err +} + +// UrlDecodeWithoutError wrapper of UrlDecode +func UrlDecodeWithoutError(value string) string { + ret, err := UrlDecode(value) + if err == nil { + return ret + } + if isErrorLogEnabled() { + doLog(LEVEL_ERROR, "Url decode error") + } + return "" +} + +// IsIP checks whether the value matches ip address +func IsIP(value string) bool { + return ipRegex.MatchString(value) +} + +// UrlEncode encodes the input value +func UrlEncode(value string, chineseOnly bool) string { + if chineseOnly { + values := make([]string, 0, len(value)) + for _, val := range value { + _value := string(val) + if regex.MatchString(_value) { + _value = url.QueryEscape(_value) + } + values = append(values, _value) + } + return strings.Join(values, "") + } + return url.QueryEscape(value) +} + +func copyHeaders(m map[string][]string) (ret map[string][]string) { + if m != nil { + ret = make(map[string][]string, len(m)) + for key, values := range m { + _values := make([]string, 0, len(values)) + for _, value := range values { + _values = append(_values, value) + } + ret[strings.ToLower(key)] = _values + } + } else { + ret = make(map[string][]string) + } + + return +} + +func parseHeaders(headers map[string][]string) (signature string, region string, signedHeaders string) { + signature = "v2" + if receviedAuthorization, ok := headers[strings.ToLower(HEADER_AUTH_CAMEL)]; ok && len(receviedAuthorization) > 0 { + if strings.HasPrefix(receviedAuthorization[0], V4_HASH_PREFIX) { + signature = "v4" + matches := v4AuthRegex.FindStringSubmatch(receviedAuthorization[0]) + if len(matches) >= 3 { + region = matches[1] + regions := regionRegex.FindStringSubmatch(region) + if len(regions) >= 2 { + region = regions[1] + } + signedHeaders = matches[2] + } + + } else if strings.HasPrefix(receviedAuthorization[0], V2_HASH_PREFIX) { + signature = "v2" + } + } + return +} + +func getTemporaryKeys() []string { + return []string{ + "Signature", + "signature", + "X-Amz-Signature", + "x-amz-signature", + } +} + +func getIsObs(isTemporary bool, querys []string, headers map[string][]string) bool { + isObs := true + if isTemporary { + for _, value := range querys { + keyPrefix := strings.ToLower(value) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + } else if strings.HasPrefix(value, HEADER_ACCESSS_KEY_AMZ) { + isObs = false + } + } + } else { + for key := range headers { + keyPrefix := strings.ToLower(key) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + break + } + } + } + return isObs +} + +func isPathStyle(headers map[string][]string, bucketName string) bool { + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + return true + } + return false +} + +// GetV2Authorization v2 Authorization +func GetV2Authorization(ak, sk, method, bucketName, objectKey, queryURL string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryURL, "?") { + queryURL = queryURL[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryURL, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + headers = copyHeaders(headers) + pathStyle := isPathStyle(headers, bucketName) + conf := &config{securityProviders: []securityProvider{NewBasicSecurityProvider(ak, sk, "")}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + conf.signature = SignatureObs + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedURL, headers, true) + v2HashPrefix := OBS_HASH_PREFIX + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + return +} + +// GetAuthorization Authorization +func GetAuthorization(ak, sk, method, bucketName, objectKey, queryURL string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryURL, "?") { + queryURL = queryURL[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryURL, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + isTemporary := false + signature := "v2" + temporaryKeys := getTemporaryKeys() + for _, key := range temporaryKeys { + if _, ok := params[key]; ok { + isTemporary = true + if strings.ToLower(key) == "signature" { + signature = "v2" + } else if strings.ToLower(key) == "x-amz-signature" { + signature = "v4" + } + break + } + } + isObs := getIsObs(isTemporary, querysResult, headers) + headers = copyHeaders(headers) + pathStyle := false + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + pathStyle = true + } + conf := &config{securityProviders: []securityProvider{NewBasicSecurityProvider(ak, sk, "")}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + + if isTemporary { + return getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature, conf, params, headers, isObs) + } + signature, region, signedHeaders := parseHeaders(headers) + if signature == "v4" { + conf.signature = SignatureV4 + requestURL, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestURL, _err := url.Parse(requestURL) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestURL") + return nil + } + headerKeys := strings.Split(signedHeaders, ";") + _headers := make(map[string][]string, len(headerKeys)) + for _, headerKey := range headerKeys { + _headers[headerKey] = headers[headerKey] + } + ret = v4Auth(ak, sk, region, method, canonicalizedURL, parsedRequestURL.RawQuery, _headers) + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedURL, headers, isObs) + v2HashPrefix := V2_HASH_PREFIX + if isObs { + v2HashPrefix = OBS_HASH_PREFIX + } + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + } + return + +} + +func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature string, conf *config, params map[string]string, + headers map[string][]string, isObs bool) (ret map[string]string) { + + if signature == "v4" { + conf.signature = SignatureV4 + + longDate, ok := params[PARAM_DATE_AMZ_CAMEL] + if !ok { + longDate = params[HEADER_DATE_AMZ] + } + shortDate := longDate[:8] + + credential, ok := params[PARAM_CREDENTIAL_AMZ_CAMEL] + if !ok { + credential = params[strings.ToLower(PARAM_CREDENTIAL_AMZ_CAMEL)] + } + + _credential := UrlDecodeWithoutError(credential) + + regions := regionRegex.FindStringSubmatch(_credential) + var region string + if len(regions) >= 2 { + region = regions[1] + } + + _, scope := getCredential(ak, region, shortDate) + + expires, ok := params[PARAM_EXPIRES_AMZ_CAMEL] + if !ok { + expires = params[strings.ToLower(PARAM_EXPIRES_AMZ_CAMEL)] + } + + signedHeaders, ok := params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] + if !ok { + signedHeaders = params[strings.ToLower(PARAM_SIGNEDHEADERS_AMZ_CAMEL)] + } + + algorithm, ok := params[PARAM_ALGORITHM_AMZ_CAMEL] + if !ok { + algorithm = params[strings.ToLower(PARAM_ALGORITHM_AMZ_CAMEL)] + } + + if _, ok := params[PARAM_SIGNATURE_AMZ_CAMEL]; ok { + delete(params, PARAM_SIGNATURE_AMZ_CAMEL) + } else if _, ok := params[strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)]; ok { + delete(params, strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)) + } + + ret = make(map[string]string, 6) + ret[PARAM_ALGORITHM_AMZ_CAMEL] = algorithm + ret[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + ret[PARAM_DATE_AMZ_CAMEL] = longDate + ret[PARAM_EXPIRES_AMZ_CAMEL] = expires + ret[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = signedHeaders + + requestURL, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestURL, _err := url.Parse(requestURL) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl") + return nil + } + stringToSign := getV4StringToSign(method, canonicalizedURL, parsedRequestURL.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, strings.Split(signedHeaders, ";"), headers) + ret[PARAM_SIGNATURE_AMZ_CAMEL] = UrlEncode(getSignature(stringToSign, sk, region, shortDate), false) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + expires, ok := params["Expires"] + if !ok { + expires = params["expires"] + } + headers[HEADER_DATE_CAMEL] = []string{expires} + stringToSign := getV2StringToSign(method, canonicalizedURL, headers, isObs) + ret = make(map[string]string, 3) + ret["Signature"] = UrlEncode(Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign))), false) + ret["AWSAccessKeyId"] = UrlEncode(ak, false) + ret["Expires"] = UrlEncode(expires, false) + } + + return +} diff --git a/cve-vulner-manager/routers/commentsRouter_controllers.go b/cve-vulner-manager/routers/commentsRouter_controllers.go new file mode 100644 index 0000000000000000000000000000000000000000..dcfa01a0d5230c42f4b65d8bafb28b78b01842a3 --- /dev/null +++ b/cve-vulner-manager/routers/commentsRouter_controllers.go @@ -0,0 +1,298 @@ +package routers + +import ( + "github.com/astaxie/beego" + "github.com/astaxie/beego/context/param" +) + +func init() { + + beego.GlobalControllerRouter["cvevulner/controllers:CveAllIssueController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveAllIssueController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:CveDetailController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveDetailController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:CveErrorFeedBackController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveErrorFeedBackController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:CveIssueInfoController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveIssueInfoController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:CveIssueWhiteListController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveIssueWhiteListController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], + beego.ControllerComments{ + Method: "DownLoadExcelByFileCode", + Router: "/downloadExcel", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], + beego.ControllerComments{ + Method: "DownloadLastExcel", + Router: "/lastExcel", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], + beego.ControllerComments{ + Method: "TriggerCveData", + Router: "/triggerCveData", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:GaussCveController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:GaussCveController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:GaussCveDetailController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:GaussCveDetailController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:GaussSaController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:GaussSaController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:GaussSaDetailController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:GaussSaDetailController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:GenSaController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:GenSaController"], + beego.ControllerComments{ + Method: "TriggerCveData", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:HookEventControllers"] = append(beego.GlobalControllerRouter["cvevulner/controllers:HookEventControllers"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:IssueOathCallbackController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:IssueOathCallbackController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:KanbanCveAllIssueController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:KanbanCveAllIssueController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "GetAll", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Get", + Router: "/:objectId", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Put", + Router: "/:objectId", + AllowHTTPMethods: []string{"put"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Delete", + Router: "/:objectId", + AllowHTTPMethods: []string{"delete"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:PackagesController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:PackagesController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:PackagesInfoController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:PackagesInfoController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "GetAll", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Get", + Router: "/:uid", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Put", + Router: "/:uid", + AllowHTTPMethods: []string{"put"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Delete", + Router: "/:uid", + AllowHTTPMethods: []string{"delete"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Login", + Router: "/login", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Logout", + Router: "/logout", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserLoginController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserLoginController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserUploadController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserUploadController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + +} diff --git a/cve-vulner-manager/routers/router.go b/cve-vulner-manager/routers/router.go new file mode 100644 index 0000000000000000000000000000000000000000..1a218a1295f394956b4b0c33898e1da088af09e3 --- /dev/null +++ b/cve-vulner-manager/routers/router.go @@ -0,0 +1,126 @@ +// @APIVersion 1.0.0 +// @Title beego Test API +// @Description beego has a very cool tools to autogenerate documents for your API +// @Contact astaxie@gmail.com +// @TermsOfServiceUrl http://beego.me/ +// @License Apache 2.0 +// @LicenseUrl http://www.apache.org/licenses/LICENSE-2.0.html +package routers + +import ( + "cvevulner/controllers" + "github.com/astaxie/beego" +) + +func init() { + ns := beego.NewNamespace("/v1", + beego.NSNamespace("/object", + beego.NSInclude( + &controllers.ObjectController{}, + ), + ), + beego.NSNamespace("/packages", + beego.NSInclude( + &controllers.PackagesController{}, + ), + ), + beego.NSNamespace("/packages/packageinfo", + beego.NSInclude( + &controllers.PackagesInfoController{}, + ), + ), + beego.NSNamespace("/user", + beego.NSInclude( + &controllers.UserController{}, + ), + ), + beego.NSNamespace("/user/login", + beego.NSInclude( + &controllers.UserLoginController{}, + ), + ), + beego.NSNamespace("/cve/upload", + beego.NSInclude( + &controllers.UserUploadController{}, + ), + ), + beego.NSNamespace("/issue/oauth/callback", + beego.NSInclude( + &controllers.IssueOathCallbackController{}, + ), + ), + beego.NSNamespace("/issue/hook/event", + beego.NSInclude(&controllers.HookEventControllers{}, + ), + ), + beego.NSNamespace("/download/excel", + beego.NSInclude( + &controllers.FileController{}, + ), + ), + beego.NSNamespace("/cve/detail", + beego.NSInclude( + &controllers.CveDetailController{}, + ), + ), + beego.NSNamespace("/cve/error/feedback", + beego.NSInclude( + &controllers.CveErrorFeedBackController{}, + ), + ), + beego.NSNamespace("/cve/issue/whitelist", + beego.NSInclude( + &controllers.CveIssueWhiteListController{}, + ), + ), + //Get the issue corresponding to the branch information affected by cve + beego.NSNamespace("/cve/issue", + beego.NSInclude( + &controllers.CveIssueInfoController{}, + ), + ), + //Obtain all cve data and provide operation kanban for use1 + beego.NSNamespace("/cve/all/issue", + beego.NSInclude( + &controllers.CveAllIssueController{}, + ), + ), + //Obtain all cve data and provide operation kanban for use2 + beego.NSNamespace("/kanban/cve/all/issue", + beego.NSInclude( + &controllers.KanbanCveAllIssueController{}, + ), + ), + // openGauss generates a summary of Sa data, and manually triggers the data at regular intervals + beego.NSNamespace("/gauss/triggerCveData", + beego.NSInclude( + &controllers.GenSaController{}, + ), + ), + // Get the information of the SA that has been generated + beego.NSNamespace("/gauss/sa", + beego.NSInclude( + &controllers.GaussSaController{}, + ), + ), + // Get the details of gauss' sa + beego.NSNamespace("/gauss/sa/detail", + beego.NSInclude( + &controllers.GaussSaDetailController{}, + ), + ), + // Get the cve information of gauss + beego.NSNamespace("/gauss/cve", + beego.NSInclude( + &controllers.GaussCveController{}, + ), + ), + // Get the cve details of gauss + beego.NSNamespace("/gauss/cve/detail", + beego.NSInclude( + &controllers.GaussCveDetailController{}, + ), + ), + ) + beego.AddNamespace(ns) +} diff --git a/cve-vulner-manager/task/cve.go b/cve-vulner-manager/task/cve.go new file mode 100644 index 0000000000000000000000000000000000000000..92a89db9c9aff995d914f5a199dbbae7080cd677 --- /dev/null +++ b/cve-vulner-manager/task/cve.go @@ -0,0 +1,66 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" +) + +//ProcCveOriginData Process raw data obtained by api +func ProcCveOriginData(prcNum, days, credibilityLevel, openeulerNum int, cveRef, owner string) (bool, error) { + // Process raw data obtained by api + ok, err := taskhandler.GetCveOriginData(prcNum, days, openeulerNum, credibilityLevel, cveRef) + if !ok { + logs.Error("ProcCveOriginData, GetCveOriginData, err: ", err) + } + // Processing excel raw data + okx, err := taskhandler.GetCveOriginExcelData(prcNum, days, openeulerNum, cveRef) + if !okx { + logs.Error("ProcCveOriginData, GetCveOriginExcelData, err: ", err) + } + return true, nil +} + +//ParamsCveOriginData Processing cve raw data +func ParamsCveOriginData() error { + defer common.Catchs() + logs.Info("Start the task of generating cve raw data into cve library.") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init, error: ", err) + return err + } + // The amount of data processed at a time + prcnum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("crontab::prcnum, error: ", err) + return errors.New("value is nil") + } + // How many days have been processed to date data + days, ok := BConfig.Int("crontab::days") + if ok != nil { + logs.Error("crontab::days, err:", err) + return ok + } + // openeuler Number start value + cveRef := BConfig.String("cve::cveref") + openeulernum, ok := BConfig.Int("cve::openeulernum") + if ok != nil { + logs.Error("cve::openeulernum, err:", err) + return ok + } + owner := BConfig.String("gitee::owner") + // cve credibility level + credibilityLevel, ok := BConfig.Int("cve::credibility_level") + if ok != nil { + logs.Error("cve::credibility_level, err:", err) + return ok + } + // Get the data source of the table + _, errx := ProcCveOriginData(prcnum, days, credibilityLevel, openeulernum, cveRef, owner) + logs.Info("The task of generating cve original data into cve library is over.") + return errx +} diff --git a/cve-vulner-manager/task/genexcel.go b/cve-vulner-manager/task/genexcel.go new file mode 100644 index 0000000000000000000000000000000000000000..48b92c2ac22816cacc2162426a6b42d52b59c021 --- /dev/null +++ b/cve-vulner-manager/task/genexcel.go @@ -0,0 +1,16 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "github.com/astaxie/beego/logs" +) + +//GenExcelData generate excel data +func GenExcelData() error{ + defer common.Catchs() + logs.Info("Execute the task of checking whether cve has already released sa...") + errx := taskhandler.GenerateExcelTask() + logs.Info("The task of executing to check whether cve has been released sa is over...") + return errx +} \ No newline at end of file diff --git a/cve-vulner-manager/task/grabissue.go b/cve-vulner-manager/task/grabissue.go new file mode 100644 index 0000000000000000000000000000000000000000..1c007d3f2e6e596c8074e7dbe40d58b8c52b104a --- /dev/null +++ b/cve-vulner-manager/task/grabissue.go @@ -0,0 +1,75 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "strconv" + "strings" +) + +//GetIssueData get the issue data +func GetIssueData() error { + defer common.Catchs() + logs.Info("The task of getting the data source of the created issue starts...") + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + communityName := beego.AppConfig.DefaultString("communityName", "src-openeuler:1") + communityList := strings.Split(communityName, ",") + for _, com := range communityList { + comList := strings.Split(com, ":") + if len(comList) == 2 { + organizationID, err := strconv.Atoi(comList[1]) + if err != nil { + logs.Error("err: ", err) + continue + } + owner, giteeToken := common.GetOwnerAndToken(int8(organizationID)) + // Synchronize gitee data to cve + errx := taskhandler.GrabIssueByOrg(giteeToken, owner) + if errx != nil { + logs.Error("errx: ", errx) + } + logs.Info("The task of obtaining the data source of the created issue ends...") + // The amount of data processed at a time + prcNum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("crontab::prcnum, error: ", err) + return errors.New("value is nil") + } + // How many days have been processed to date data + days, ok := BConfig.Int("crontab::days") + if ok != nil { + logs.Error("crontab::days, err:", err) + return ok + } + // openeuler Number start value + cveRef := BConfig.String("cve::cveref") + openeulerNum, ok := BConfig.Int("cve::openeulernum") + if ok != nil { + logs.Error("cve::openeulernum, err:", err) + return ok + } + logs.Info("Start syncing gitee data to the cve vulnerability database") + // Compatible with created issue data + oki, err := taskhandler.GetCveIssueData(prcNum, days, openeulerNum, cveRef, owner, 2) + if !oki { + logs.Error("ProcCveOriginData, GetCveIssueData, err: ", err) + } + // Synchronous Data + duplicationDate, ok := BConfig.Int("cve::de_duplication_date") + if ok != nil { + duplicationDate = -30 + } + taskhandler.ProcIssueTemplateBranch(duplicationDate) + } + } + logs.Info("End of syncing gitee data to cve vulnerability database") + return err +} diff --git a/cve-vulner-manager/task/hook.go b/cve-vulner-manager/task/hook.go new file mode 100644 index 0000000000000000000000000000000000000000..8b308a5991262e0ad50c96f4723fa6dcf2f0d476 --- /dev/null +++ b/cve-vulner-manager/task/hook.go @@ -0,0 +1,52 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "os" +) + +func ProcHookEvent() error { + defer common.Catchs() + logs.Info("Remove duplicate webhook event task start") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + hookurl := BConfig.String("hook::hookurl") + owner := BConfig.String("gitee::owner") + accessToken := os.Getenv("GITEE_TOKEN") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") + pwd := BConfig.String("hook::hookpwd") + // Get the data source of the table + err = taskhandler.ProcHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken) + logs.Info("Remove duplicate webhook event task ends") + return err +} + +// Create webhooks for all repo +func CreateWebhookEvent() error { + defer common.Catchs() + logs.Info("Create webhook event start") + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + hookurl := BConfig.String("hook::hookurl") + owner := BConfig.String("gitee::owner") + accessToken := os.Getenv("GITEE_TOKEN") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") + pwd := BConfig.String("hook::hookpwd") + // Get the data source of the table + err = taskhandler.CreateHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken) + logs.Info("End of creating webhook event") + return err +} diff --git a/cve-vulner-manager/task/inittask.go b/cve-vulner-manager/task/inittask.go new file mode 100644 index 0000000000000000000000000000000000000000..2a22a4250c382d033e43f9ab68a46fd42e96348e --- /dev/null +++ b/cve-vulner-manager/task/inittask.go @@ -0,0 +1,335 @@ +package task + +import ( + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/toolbox" +) + +//CheckOriCveTask Verify the original cve data +func CheckOriCveTask(oriCveCheck string) { + logs.Info("The task of verifying the original cve data starts...") + CheckTask := toolbox.NewTask("CheckOriCve", oriCveCheck, CheckOriCve) + toolbox.AddTask("CheckOriCve", CheckTask) + logs.Info("End of verifying original cve data task...") +} + +//InitYamlTask Get yaml data source +func InitYamlTask(getYaml string) { + logs.Info("Get the yaml data source task started...") + yamlTask := toolbox.NewTask("GetYamlData", getYaml, GetYamlData) + toolbox.AddTask("GetYamlData", yamlTask) + logs.Info("End of the task of obtaining yaml data source...") +} + +//InitEulerYamlTask Get yaml data source +func InitEulerYamlTask(eulergetymal string) { + logs.Info("Get the euleryaml data source task started...") + eulerYamlTask := toolbox.NewTask("GetEulerYamlData", eulergetymal, GetEulerYamlData) + toolbox.AddTask("GetEulerYamlData", eulerYamlTask) + logs.Info("End of the task of obtaining euleryaml data source...") +} + +//InitCveTask Generate cve original data into cve library +func InitCveTask(getCve string) { + logs.Info("The task of generating cve vulnerability library from cve raw data begins...") + CveTask := toolbox.NewTask("ParamsCveOriginData", getCve, ParamsCveOriginData) + toolbox.AddTask("ParamsCveOriginData", CveTask) + logs.Info("The task of generating cve vulnerability library from cve raw data ends...") +} + +// InitIssueTask Get the created issue data source +func InitIssueTask(getIssue string) { + logs.Info("Get the created issue data source task starts...") + issueTask := toolbox.NewTask("GetIssueData", getIssue, GetIssueData) + toolbox.AddTask("GetIssueData", issueTask) + logs.Info("End of the task of obtaining the created issue data source...") +} + +// InitIssueToken Execute the token acquisition task to start +func InitIssueToken(issueOath string) { + logs.Info("Execute the token acquisition task to start...") + TokenTask := toolbox.NewTask("GetGiteeToken", issueOath, GetGiteeToken) + toolbox.AddTask("GetGiteeToken", TokenTask) + logs.Info("End of execution of the token acquisition task...") +} + +// CreatTask Execute the create issue task to start +func CreatTask(createIssue string) { + logs.Info("Execute the create issue task to start...") + cIssueTask := toolbox.NewTask("CreateIssue", createIssue, CreateIssue) + toolbox.AddTask("CreateIssue", cIssueTask) + logs.Info("End of execution of the task of creating an issue...") +} + +// Handle the current emergency cve and create an issue +func EmgCreatTask(emergCreateIssue string) { + logs.Info("Handle the current emergency cve and create an issue task to start...") + cIssueTask := toolbox.NewTask("EmgCreateIssue", emergCreateIssue, EmgCreateIssue) + toolbox.AddTask("EmgCreateIssue", cIssueTask) + logs.Info("Handle the current emergency cve and create an issue to end...") +} + +// unlock table +func UnlockTask(unLockTable string) { + logs.Info("Execute unlock table task start...") + cUnLockTable := toolbox.NewTask("UnLockTable", unLockTable, UnLockTable) + toolbox.AddTask("UnLockTable", cUnLockTable) + logs.Info("End of execution of unlock table task...") +} + +// Handling hook events +func ProcHookTask(prochook string) { + logs.Info("Execute the Hook event task start...") + cProchook := toolbox.NewTask("ProcHookEvent", prochook, ProcHookEvent) + toolbox.AddTask("ProcHookEvent", cProchook) + logs.Info("The execution of the Hook event task ends...") +} + +// Create webhooks for all repo +func CreateHookTask(createHook string) { + logs.Info("Start creating all webhook tasks...") + createhook := toolbox.NewTask("CreateWebhookEvent", createHook, CreateWebhookEvent) + toolbox.AddTask("CreateWebhookEvent", createhook) + logs.Info("End of execution to create all webhook tasks...") +} + +//GenSAExcelTask Execute issue to generate excel task start +func GenSAExcelTask(genExcel string) { + logs.Info("Execute issue to generate excel task start...") + genExcelTask := toolbox.NewTask("GenExcelData", genExcel, GenExcelData) + toolbox.AddTask("GenExcelData", genExcelTask) + logs.Info("Execute issue to generate excel task ends...") +} + +// Handling abnormal cve data +func AbnormalCveTask(exceptcve string) { + logs.Info("Execute abnormal cve task task start...") + procabnTask := toolbox.NewTask("AbnormalCve", exceptcve, ProcAbnCve) + toolbox.AddTask("AbnormalCve", procabnTask) + logs.Info("Execute processing exception cve task task end...") +} + +// Generate SA number +func GenerateSaNumberTask(genSa string) { + logs.Info("Start the task of generating SA number...") + proSaTask := toolbox.NewTask("GenerateSaNumber", genSa, ProcSaNumber) + toolbox.AddTask("GenerateSaNumber", proSaTask) + logs.Info("The task of generating SA number is over...") +} + +// issue statistics mailing list +func IssueStstisticsTask(issueStatistic string) { + logs.Info("Execute issue statistics task start...") + iStTask := toolbox.NewTask("IssueStatistics", issueStatistic, IssueStatistics) + toolbox.AddTask("IssueStatistics", iStTask) + logs.Info("The end of the issue statistics task...") +} + +// Cve data email task with low credibility +func cveLowCredTask(cvecredit string) { + logs.Info("Execute the CVE credibility data feedback task started...") + iStTask := toolbox.NewTask("ProcCveLowCred", cvecredit, ProcCveLowCred) + toolbox.AddTask("ProcCveLowCred", iStTask) + logs.Info("The end of the CVE credibility data feedback task...") +} + +// Handling urgent tasks +func UrgentTask(urgenttask string) { + logs.Info("Handling urgent task started...") + iStTask := toolbox.NewTask("HandUrgentData", urgenttask, HandUrgentData) + toolbox.AddTask("HandUrgentData", iStTask) + logs.Info("The end of the Handling urgent task...") +} + +// Add security bulletin link task +func SecurityLinkTask(seclinktask string) { + logs.Info("Add security bulletin link task started...") + sEtTask := toolbox.NewTask("ProcSecLinkTemplate", seclinktask, ProcSecLinkTemplate) + toolbox.AddTask("ProcSecLinkTemplate", sEtTask) + logs.Info("End of task of adding a link to a security bulletin...") +} + +// Verify whether the issue on gitee is deleted, and the data is synchronized +func CheckIssueDel(checkissuedel string) { + logs.Info("Verify whether the issue on gitee is deleted and the task starts...") + cHiTask := toolbox.NewTask("CheckIssueIsExist", checkissuedel, CheckIssueIsExist) + toolbox.AddTask("CheckIssueIsExist", cHiTask) + logs.Info("Verify whether the issue on gitee is deleted and the task ends...") +} + +// Print logs to the console and delete redundant logs +func PrintLogTask(printLog string) { + logs.Info("Execute log task task start...") + printLogTask := toolbox.NewTask("ProcLogData", printLog, ProcLogData) + toolbox.AddTask("ProcLogData", printLogTask) + logs.Info("Execute log task task end...") +} + +//Statistics of issues created in different communities, uncreated issues are created +func IssueCommunityStatistTask(issuecommunity string) { + logs.Info("Community issue statistics task started...") + issueStatTask := toolbox.NewTask("CommunityIssueStatist", issuecommunity, CommunityIssueStatist) + toolbox.AddTask("CommunityIssueStatist", issueStatTask) + logs.Info("Community issue statistics task is over...") +} + +//Check whether the issue label is reasonable +func IssueLabelCheckTask(issuelabelcheck string) { + issueLaCheckTask := toolbox.NewTask("IssueLabelCheck", issuelabelcheck, IssueLabelCheck) + toolbox.AddTask("IssueLabelCheck", issueLaCheckTask) +} + +// start task +func StartTask() { + toolbox.StartTask() +} + +func StopTask() { + toolbox.StopTask() +} + +//InitTask Timing task initialization +func InitTask() bool { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error: file:conf/app.conf: ", err) + return false + } + // Get the original yaml data + ymalflag, errxs := BConfig.Int("crontab::ymalflag") + if ymalflag == 1 && errxs == nil { + getymal := BConfig.String("crontab::getymal") + InitYamlTask(getymal) + } + // Verify the original cve data + oricveflag, errxs := BConfig.Int("crontab::oricveflag") + if oricveflag == 1 && errxs == nil { + oricvecheck := BConfig.String("crontab::oricvecheck") + CheckOriCveTask(oricvecheck) + } + // Refresh gittoken regularly + gittokenflag, errx := BConfig.Int("crontab::gittokenflag") + if gittokenflag == 1 && errx == nil { + issueoath := BConfig.String("crontab::issueoath") + // Get token from environment variable + gitToken := BConfig.String("gitee::git_token") + if gitToken == "xxx" { + InitIssueToken(issueoath) + } + } + // Get the created issue + getissueflag, errxs := BConfig.Int("crontab::getissueflag") + if getissueflag == 1 && errxs == nil { + getissue := BConfig.String("crontab::getissue") + InitIssueTask(getissue) + } + // Generate cve vulnerability library + cveflag, errxs := BConfig.Int("crontab::cveflag") + if cveflag == 1 && errxs == nil { + getcve := BConfig.String("crontab::getcve") + InitCveTask(getcve) + } + // Regularly output logs to the console + printLogFlag, errxs := BConfig.Int("crontab::printlogflag") + if printLogFlag == 1 && errxs == nil { + printLog := BConfig.String("crontab::printlog") + PrintLogTask(printLog) + } + // Add issue + taskFlag, errx := BConfig.Int("crontab::issueflag") + if taskFlag == 1 && errx == nil { + createIssue := BConfig.String("crontab::createissue") + CreatTask(createIssue) + } + // Handle the current emergency cve and create an issue + emergissueflag, errx := BConfig.Int("crontab::emergissueflag") + if emergissueflag == 1 && errx == nil { + emergCreateIssue := BConfig.String("crontab::emergcreateissue") + EmgCreatTask(emergCreateIssue) + } + // Generate excel data from issue + genexcelflag, errx := BConfig.Int("crontab::genexcelflag") + if genexcelflag == 1 && errx == nil { + genExcel := BConfig.String("crontab::genexcel") + GenSAExcelTask(genExcel) + } + // unlock mysql + unlockflag, errx := BConfig.Int("crontab::unlockflag") + if unlockflag == 1 && errx == nil { + unlock := BConfig.String("crontab::unlock") + UnlockTask(unlock) + } + // Handling hook events + hookflag, errx := BConfig.Int("crontab::hookflag") + if hookflag == 1 && errx == nil { + prochook := BConfig.String("crontab::prochook") + ProcHookTask(prochook) + } + // Create webhooks for all repo + createHookFlag, errx := BConfig.Int("crontab::createhookflag") + if createHookFlag == 1 && errx == nil { + createHook := BConfig.String("crontab::createhook") + CreateHookTask(createHook) + } + // Handling abnormal cve data + exceptflag, errx := BConfig.Int("crontab::exceptflag") + if exceptflag == 1 && errx == nil { + exceptcve := BConfig.String("crontab::exceptcve") + AbnormalCveTask(exceptcve) + } + // Generate SA number + gensaflag, errx := BConfig.Int("crontab::gensaflag") + if gensaflag == 1 && errx == nil { + gensa := BConfig.String("crontab::gensa") + GenerateSaNumberTask(gensa) + } + // issue statistics mailing list + issuestatisticflag, errx := BConfig.Int("crontab::issuestatisticflag") + if issuestatisticflag == 1 && errx == nil { + issueStatistic := BConfig.String("crontab::issuestatistic") + IssueStstisticsTask(issueStatistic) + } + // Cve data email task with low credibility + cvecredflag, errx := BConfig.Int("crontab::cvecredflag") + if cvecredflag == 1 && errx == nil { + cvecredit := BConfig.String("crontab::cvecredit") + cveLowCredTask(cvecredit) + } + // Handling urgent tasks + urgenttaskflag, errx := BConfig.Int("crontab::urgenttaskflag") + if urgenttaskflag == 1 && errx == nil { + urgenttask := BConfig.String("crontab::urgenttask") + UrgentTask(urgenttask) + } + // Add security bulletin link task + seclinkflag, errx := BConfig.Int("crontab::seclinkflag") + if seclinkflag == 1 && errx == nil { + seclinktask := BConfig.String("crontab::seclinktask") + SecurityLinkTask(seclinktask) + } + // Verify whether the issue on gitee is deleted, and the data is synchronized + checkissuedelflag, errx := BConfig.Int("crontab::checkissuedelflag") + if checkissuedelflag == 1 && errx == nil { + checkissuedel := BConfig.String("crontab::checkissuedel") + CheckIssueDel(checkissuedel) + } + // Obtain openEuler data sources from other channels + eulerymalflag, errxs := BConfig.Int("crontab::eulerymalflag") + if eulerymalflag == 1 && errxs == nil { + eulergetymal := BConfig.String("crontab::eulergetymal") + InitEulerYamlTask(eulergetymal) + } + // Statistics of issues created in different communities, uncreated issues are created + issuecommunityflag, errxs := BConfig.Int("crontab::issuecommunityflag") + if issuecommunityflag == 1 && errxs == nil { + issuecommunity := BConfig.String("crontab::issuecommunity") + IssueCommunityStatistTask(issuecommunity) + } + issuelabelcheckflag, errxs := BConfig.Int("crontab::issuelabelcheckflag") + if issuelabelcheckflag == 1 && errxs == nil { + issuelabelcheck := BConfig.String("crontab::issuelabelcheck") + IssueLabelCheckTask(issuelabelcheck) + } + return true +} diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go new file mode 100644 index 0000000000000000000000000000000000000000..d25e302f63cc82591d0e9652fa62dea5ec1f9ae7 --- /dev/null +++ b/cve-vulner-manager/task/issue.go @@ -0,0 +1,199 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/taskhandler" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "strconv" + "strings" +) + +// Verify whether the issue on gitee has been deleted +func CheckIssueIsExist() error { + defer common.Catchs() + prnum, prErr := beego.AppConfig.Int("crontab::prcnum") + if prErr != nil { + prnum = 100 + } + // Get the data source of the table + err := taskhandler.ProcIssueIsExists(prnum) + logs.Info(err) + return err +} + +// Statistics of issues created in different communities, uncreated issues are created +func CommunityIssueStatist() error { + // Get the configuration file + yamlConfigTable := beego.AppConfig.String("cve::yaml_config_table") + tableList := strings.Split(yamlConfigTable, ",") + localList := make([][]string, 0) + cyc := models.GetCommunityYamlConfigAll() + if len(cyc) > 0 { + for _, c := range cyc { + taskhandler.StatistCommunityIssue(c) + } + } + for _, tab := range tableList { + flag := false + tabList := strings.Split(tab, ":") + for _, c := range cyc { + if tabList[0] == c.TableName { + flag = true + break + } + } + if !flag { + if len(tabList) > 1 { + localList = append(localList, tabList) + } + } + } + if len(localList) > 0 { + for _, lc := range localList { + organizationID, _ := strconv.Atoi(lc[1]) + cyc := models.CommunityYamlConfig{TableName: lc[0], OrganizationID: int8(organizationID)} + num, err := models.InsertCommunityYamlConfig(&cyc) + if err != nil { + logs.Error("num: ", num, ",err: ", err) + } + } + } + return nil +} + +func IssueLabelCheck() error { + uNaffected := beego.AppConfig.String("labeUnaffected") + labelFixed := beego.AppConfig.String("labelFixed") + labelUnFix := beego.AppConfig.String("labelUnFix") + templateId := int64(0) + num := 100 + for { + issueTemp, issueErr := models.GetIssueTemplateByStatus(3, templateId, num) + if len(issueTemp) == 0 { + logs.Error("issueErr", issueErr) + return issueErr + } + for _, issue := range issueTemp { + templateId = issue.TemplateId + path := issue.Repo + if issue.StatusName != "closed" && issue.StatusName != "已完成" { + logs.Error("issue.StatusName: ", issue.StatusName) + continue + } + affectBrandFlag := false + cveCenter := models.VulnCenter{CveId: issue.CveId, CveNum: issue.CveNum} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if cveErr != nil { + logs.Error("cveErr: ", cveErr) + continue + } + owner, token := common.GetOwnerAndToken(cveCenter.OrganizationID) + sn := models.SecurityNotice{CveId: issue.CveId, CveNum: issue.CveNum} + snErr := sn.Read("cve_id", "cve_num") + if snErr != nil { + logs.Error("snErr: ", snErr) + } else { + affectBranchsxList := make([]string, 0) + affectedBranchs := "" + if cveCenter.OrganizationID == 4 { + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + } else if cveCenter.OrganizationID == 3 { + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + tmpTagList := make([]string, 0) + affectProductSlice := strings.Split(sn.AffectProduct, "/") + for _, tags := range affectProductSlice { + mdbt := models.MindSporeBrandTags{PackageName: path, Tags: tags} + mtErr := models.QueryMindSporeBrandTags(&mdbt, "PackageName", "Tags") + if mtErr == nil { + tmpTagList = append(tmpTagList, mdbt.Brand) + } + } + if len(tmpTagList) > 0 { + sn.AffectProduct = strings.Join(tmpTagList, "/") + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + } else { + sn.AffectProduct = "" + } + } + } else if cveCenter.OrganizationID == 2 { + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") + } else { + affectedBranchs = beego.AppConfig.String("cve::affected_branchs") + } + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + affectProductList := strings.Split(sn.AffectProduct, "/") + if len(affectProductList) > 0 { + for _, brands := range affectProductList { + if len(affectBranchsxList) > 0 { + for _, affectBranch := range affectBranchsxList { + if affectBranch == brands { + affectBrandFlag = true + break + } + } + } + if affectBrandFlag { + break + } + } + } + } + switch issue.IssueStatus { + case 2: + sn.AffectStatus = "Fixed" + case 6: + sn.AffectStatus = "UnAffected" + default: + sn.AffectStatus = "UnFixed" + } + err := sn.Update("affect_status") + if err != nil { + logs.Error(err) + } + } + if affectBrandFlag { + issue.IssueLabel = labelFixed + } else { + issue.IssueLabel = uNaffected + } + labelSlice := make([]string, 0) + labelSlice = taskhandler.AddLabelValue(token, path, issue.IssueNum, owner, issue.IssueLabel, 2) + upLabelFlag := false + if len(labelSlice) > 0 { + labelStr := strings.Join(labelSlice, ",") + if affectBrandFlag { + if strings.Contains(labelStr, uNaffected) { + labelSlice = common.DeleteSliceValue(labelSlice, uNaffected) + upLabelFlag = true + } + } else { + if strings.Contains(labelStr, labelFixed) { + labelSlice = common.DeleteSliceValue(labelSlice, labelFixed) + upLabelFlag = true + } + } + if strings.Contains(labelStr, labelUnFix) { + labelSlice = common.DeleteSliceValue(labelSlice, labelUnFix) + upLabelFlag = true + } + issue.IssueLabel = strings.Join(labelSlice, ",") + issue.CveLevel = models.OpenEulerScoreProc(issue.OpenEulerScore) + tpErr := models.UpdateIssueTemplate(&issue, "IssueLabel", "CveLevel") + if tpErr != nil { + logs.Error(tpErr) + } + } + if upLabelFlag { + update := taskhandler.UpdateIssueLabels(token, path, issue.IssueNum, owner, issue.IssueLabel) + if !update { + logs.Error("update gitee issue label fail ,", issue.IssueNum, issue.IssueLabel) + } + } + } + } +} diff --git a/cve-vulner-manager/task/issuestatistics.go b/cve-vulner-manager/task/issuestatistics.go new file mode 100644 index 0000000000000000000000000000000000000000..4ac7b6020835d92e25ba5b76c68ac7527b564c77 --- /dev/null +++ b/cve-vulner-manager/task/issuestatistics.go @@ -0,0 +1,76 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "os" +) + +// issue statistics mailing list +func IssueStatistics() error { + defer common.Catchs() + logs.Info("issue statistics task start") + // The number of days the issue counts the mailing list + beforeMonth, beErr := beego.AppConfig.Int("cve::before_month") + if beErr != nil { + beforeMonth = -3 + } + prnum, prErr := beego.AppConfig.Int("crontab::prcnum") + if prErr != nil { + prnum = 100 + } + owner := beego.AppConfig.String("gitee::owner") + // Get the data source of the table + err := taskhandler.IssueStatistics(beforeMonth, prnum, owner) + logs.Info(err) + logs.Info("issue statistics task end") + return err +} + +// Cve data email task with low credibility +func ProcCveLowCred() error { + defer common.Catchs() + logs.Info("cve credibility task start") + // The number of days the issue counts the mailing list + beforeDate, beErr := beego.AppConfig.Int("cve::cve_statistics_date") + if beErr != nil { + beforeDate = 30 + } + prnum, prErr := beego.AppConfig.Int("crontab::prcnum") + if prErr != nil { + prnum = 100 + } + // Get the data source of the table + err := taskhandler.CveCredibilityStatistics(beforeDate, prnum) + logs.Info(err) + logs.Info("cve credibility task end") + return err +} + +func ProcSecLinkTemplate() error { + defer common.Catchs() + logs.Info("Add security bulletin link task started") + // The number of days the issue counts the mailing list + beforeDate, beErr := beego.AppConfig.Int("cve::sec_link_date") + if beErr != nil { + beforeDate = -30 + } + prnum, prErr := beego.AppConfig.Int("crontab::prcnum") + if prErr != nil { + prnum = 100 + } + owner := beego.AppConfig.String("gitee::owner") + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("ProcSecLinkTemplate, issue token Get failed, current time: ", common.GetCurTime()) + return errors.New("Failed to get token in environment variable") + } + // Get the data source of the table + err := taskhandler.ProcSecLinkTemplate(beforeDate, prnum, owner, accessToken) + logs.Info(err) + logs.Info("End of task of adding a link to a security bulletin") + return err +} diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go new file mode 100644 index 0000000000000000000000000000000000000000..609acf80f862f7a062ce19784a0f033c1420c95c --- /dev/null +++ b/cve-vulner-manager/task/issuetask.go @@ -0,0 +1,883 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/taskhandler" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "strconv" + "strings" + "sync" + "time" +) + +var mutex sync.Mutex + +//GetGiteeToken Get Code Cloud API TOKEN +func GetGiteeToken() error { + defer common.Catchs() + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("GetGiteeToken, config init error:", err) + return err + } + clientID := BConfig.String("gitee::client_id") + if clientID == "" { + logs.Error("GetGiteeToken, config gitee::clientID ,error: invalid value") + return errors.New("value is nil") + } + clientSecret := beego.AppConfig.String("gitee::client_secret") + if clientSecret == "" { + logs.Error("GetGiteeToken, config gitee::clientSecret, error: invalid value") + return errors.New("value is nil") + } + Email := BConfig.String("gitee::email") + if Email == "" { + logs.Error("GetGiteeToken, config gitee::email, error: value is nil") + return errors.New("value is nil") + } + password := BConfig.String("gitee::password") + if password == "" { + logs.Error("GetGiteeToken, config gitee::password, error: value is nil") + return errors.New("value is nil") + } + Scope := BConfig.String("gitee::scope") + if Scope == "" { + logs.Error("GetGiteeToken, config gitee::scope, error: value is nil") + return errors.New("value is nil") + } + logs.Info(clientID, clientSecret, password) + var gt taskhandler.GiteeToken + gt.GrantType = "password" + gt.UserName = Email + gt.Password = password + gt.ClientID = clientID + gt.ClientSecret = clientSecret + gt.Scope = Scope + taskhandler.GetOautToken(gt) + return nil +} + +// Assemble issue record data +func GenIssueRecordData(icr *models.IssueCreateRecord, issueValue models.VulnCenter, sc models.Score, status int8) { + icr.CveId = issueValue.CveId + icr.CveNum = issueValue.CveNum + icr.Description = issueValue.Description + icr.CveLevel = issueValue.CveLevel + icr.Status = status + icr.CveVersion = issueValue.CveVersion + icr.RepairTime = issueValue.RepairTime + icr.PackName = issueValue.PackName + icr.NVDScore = sc.NVDScore + icr.NvectorVule = sc.NvectorVule + icr.CreateTime = common.GetCurTime() +} + +func SaveCveTempError(errDesc, cveNum string, cveId int64) { + if len(errDesc) > 1 { + models.UpdateTempleteError(errDesc, cveNum, cveId) + } +} + +func ErrorCveStatistics(errDesc string, issueValue models.VulnCenter, status int8) { + SaveCveTempError(errDesc, issueValue.CveNum, issueValue.CveId) + if issueValue.OrganizationID == 1 { + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("ErrorCveStatistics, Failed to get Score, err: ", err, ", cveId: ", issueValue.CveId) + return + } + icr := models.IssueCreateRecord{ErrorDescription: errDesc} + GenIssueRecordData(&icr, issueValue, sc, status) + var it models.IssueTemplate + it.CveId = issueValue.CveId + it.CveNum = issueValue.CveNum + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if templateErr == nil { + if len(it.CreateTime.String()) > 19 { + icr.CreateTime = it.CreateTime.String()[:19] + } else { + icr.CreateTime = it.CreateTime.String()[:19] + } + } + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("ErrorCveStatistics, Failed to create issue record data, err: ", issReErr) + } + } +} + +func CheckCveIssueBranch(packageName, version string) bool { + submitIssueBranch := beego.AppConfig.String("cve::submit_issue_branch") + submitIssueBranchSlice := strings.Split(submitIssueBranch, ",") + branchFlag := false + if len(version) > 0 { + versionSlice := strings.Split(version, ",") + if len(versionSlice) > 0 { + for _, ver := range versionSlice { + var oro models.OpenEulerRepoOrigin + oro.PackageName = packageName + oro.Version = ver + _ = models.QueryEulerRepoOrigin(&oro, "PackageName", "Version") + if oro.Id > 0 { + if len(submitIssueBranchSlice) > 0 && len(oro.Branchs) > 1 { + BranchSlice := strings.Split(oro.Branchs, ",") + if len(BranchSlice) > 0 { + for _, confBr := range submitIssueBranchSlice { + for _, br := range BranchSlice { + if confBr == br { + branchFlag = true + goto compareRes + } + } + } + } + } + } + } + } + } + return branchFlag +compareRes: + return branchFlag +} + +func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, flag int) error { + for ; ; { + cveData, err := models.QueryIssue(beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info("cveData: ", cveData) + //ss := rand.Int31n(10) + //time.Sleep(time.Second * time.Duration(ss)) + } else { + logs.Info("addUnlimitedIssue, No cve data can be used, current time: ", common.GetCurTime(), ", err: ", err) + return err + } + for index, issueValue := range cveData { + logs.Info("addUnlimitedIssue, Currently processing：", index, ",cve data, cveNum: ", issueValue.CveNum) + // add mutex + lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) + if !lockErr { + logs.Error("addUnlimitedIssue, The current cve is processing, continue to process the next data, "+ + "err: ", lockErr, ",data: ", issueValue) + continue + } + owner, accessToken := common.GetOwnerAndToken(issueValue.OrganizationID) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" || goi.IssueState == "已挂起" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addUnlimitedIssue, The cve data has already been submitted to the issue, "+ + "no need to submit repeatedly, cveData: ", issueValue) + ErrorCveStatistics("CVE已创建过issue, 且已归档", issueValue, 1) + continue + } + } + if issueValue.OrganizationID == 1 { + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ + "it has been processed, cveData: ", issueValue) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) + continue + } + // Determine whether cve has been processed + issueExist, _ := taskhandler.GetCveSecurityNotice(issueValue.CveNum, issueValue.PackName) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addUnlimitedIssue, The cve data has been displayed on the official website, "+ + "no need to submit an issue on git, cveData: ", issueValue) + ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + continue + } + if issueValue.Status == 0 && flag != 2 && len(issueValue.CveVersion) > 0 { + branchBool := CheckCveIssueBranch(issueValue.PackName, issueValue.CveVersion) + if !branchBool { + models.UpdateIssueStatus(issueValue, 8) + logs.Info("addUnlimitedIssue, "+ + "The current repo branch and version information do not match, "+ + "cveData: ", issueValue) + ErrorCveStatistics("仓库分支和版本信息不想符合", issueValue, 2) + continue + } + } + } else if issueValue.OrganizationID == 2 { + ogc := models.OpenGaussCveList{CveNum: issueValue.CveNum, PackName: issueValue.RepoName, Status: 3} + ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName", "Status") + if ogc.Id > 0 { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addUnlimitedIssue, The cve data has been displayed on the official website, "+ + "no need to submit an issue on git, cveData: ", issueValue, ogcErr) + ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + continue + } + } + var it models.IssueTemplate + it.CveId = issueValue.CveId + it.CveNum = issueValue.CveNum + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if templateErr != nil { + logs.Warn("addUnlimitedIssue, templateErr:", templateErr, ", CveNum: ", issueValue.CveNum) + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears + } + if cveYears <= years { + yearFlag := false + if it.TemplateId > 0 && len(it.IssueNum) > 0 { + issueErr, issueBody := taskhandler.GetGiteeIssue(accessToken, owner, issueValue.PackName, it.IssueNum) + if issueErr == nil && len(issueBody) > 0 { + yearFlag = true + } + } + if !yearFlag { + models.UpdateIssueStatus(issueValue, 4) + logs.Info("addUnlimitedIssue, cve: ", issueValue.CveNum, ",Need to be greater than： ", + years, ",Otherwise, there is no need to submit an issue on git, cveData: ", issueValue) + ErrorCveStatistics("CVE年限受限", issueValue, 2) + continue + } + } + } + } + // Process each piece of cve data + if issueValue.Status == 0 || len(it.IssueNum) < 2 { + issueValue.Status = 2 + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner) + mutex.Unlock() + if err != nil { + logs.Error("addUnlimitedIssue, Failed to create issue, cvenum: ", + issueValue.CveNum, "err,err: ", err) + errDesc := fmt.Sprintf("%v", err) + ErrorCveStatistics(errDesc, issueValue, 2) + continue + } + } else { + issueValue.Status = 2 + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner) + mutex.Unlock() + if err != nil { + logs.Error("addUnlimitedIssue, Failed to update issue, cvenum: ", + issueValue.CveNum, "err,err: ", err) + errDesc := fmt.Sprintf("%v", err) + ErrorCveStatistics(errDesc, issueValue, 2) + continue + } + } + // Collect issue record data + ErrorCveStatistics("success", issueValue, 1) + } + } +} + +func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + issueWhiteData, issueErr := models.QueryIssueWhitelist(0) + if issueErr == nil && len(issueWhiteData) > 0 { + for _, issues := range issueWhiteData { + if issues.Status == 2 { + logs.Error("addLimitedIssue, Removed from whitelist, repo: ", issues.PackageName) + continue + } + for ; ; { + cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("addLimitedIssue, No cve data can be used, "+ + "current time: ", common.GetCurTime(), ", err: ", err) + break + } + for index, issueValue := range cveData { + logs.Info("addLimitedIssue, Currently processing section: ", index, + ", Cve data, cveNum: ", issueValue.CveNum) + // add mutex + lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) + if !lockErr { + logs.Error("addLimitedIssue, The current cve is processing, continue to"+ + " process the next data, err: ", lockErr, ",data: ", issueValue) + continue + } + owner, accessToken := common.GetOwnerAndToken(issueValue.OrganizationID) + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" || goi.IssueState == "已挂起" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addLimitedIssue, The cve data has already been submitted to the issue, "+ + "no need to submit repeatedly, cveData: ", issueValue) + ErrorCveStatistics("CVE已创建过issue, 且已归档", issueValue, 1) + continue + } + } + // Determine whether the issue has been processed + if issueValue.OrganizationID == 1 { + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, + PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("addLimitedIssue, The current issue does not need to be processed,"+ + " it has been processed, cveData: ", issueValue) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) + continue + } + // Determine whether cve has been processed + issueExist, _ := taskhandler.GetCveSecurityNotice(issueValue.CveNum, issueValue.PackName) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addLimitedIssue, The cve data has been displayed on the official website, "+ + "no need to submit an issue on git, cveData: ", issueValue) + ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + continue + } + branchBool := CheckCveIssueBranch(issueValue.PackName, issueValue.CveVersion) + if !branchBool { + models.UpdateIssueStatus(issueValue, 8) + logs.Info("addLimitedIssue, The current repo branch and version information do not match, cveData: ", issueValue) + ErrorCveStatistics("仓库分支和版本信息不想符合", issueValue, 2) + continue + } + } else if issueValue.OrganizationID == 2 { + ogc := models.OpenGaussCveList{CveNum: issueValue.CveNum, PackName: issueValue.PackName, Status: 3} + ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName", "Status") + if ogc.Id > 0 { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addLimitedIssue, The cve data has been displayed on the official website, "+ + "no need to submit an issue on git, cveData: ", issueValue, ogcErr) + ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears + } + if cveYears <= years { + models.UpdateIssueStatus(issueValue, 4) + logs.Info("addLimitedIssue, cve: ", issueValue.CveNum, ",Need to be greater than: ", + years, ", Otherwise there is no need to submit an issue on git, cveData: ", issueValue) + ErrorCveStatistics("CVE年限受限", issueValue, 2) + continue + } + } + } + // Process each piece of cve data + if issueValue.Status == 0 { + issueValue.Status = 2 + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner) + mutex.Unlock() + if err != nil { + logs.Error("addLimitedIssue, Failed to create issue, cvenum: ", + issueValue.CveNum, ",err: ", err) + errDesc := fmt.Sprintf("%v", err) + ErrorCveStatistics(errDesc, issueValue, 2) + continue + } + } else { + issueValue.Status = 2 + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner) + mutex.Unlock() + if err != nil { + logs.Error("addLimitedIssue, Failed to modify the issue, cvenum: ", + issueValue.CveNum, "err,err: ", err) + errDesc := fmt.Sprintf("%v", err) + ErrorCveStatistics(errDesc, issueValue, 2) + continue + } + } + // Collect issue record data + ErrorCveStatistics("success", issueValue, 1) + } + } + } + } else { + logs.Error(issueErr) + return issueErr + } + return nil +} + +//CreateIssue Create issue +func CreateIssue() error { + defer common.Catchs() + logs.Info("The issue task of creating cve begins. . .") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("CreateIssue, config init, error:", err) + return err + } + days, ok := BConfig.Int("crontab::days") + if ok != nil { + logs.Error("CreateIssue, config crontab::days, error:", ok) + return ok + } + years, ok := BConfig.Int("cve::cve_number") + if ok != nil { + logs.Error("CreateIssue, config cve::years, error:", ok) + return ok + } + manYears, ok := BConfig.Int("cve::cve_number_m") + if ok != nil { + logs.Error("CreateIssue, config cve::manYears, error:", ok) + return ok + } + toolYears, ok := BConfig.Int("cve::cve_number_t") + if ok != nil { + logs.Error("CreateIssue, config cve::toolYears, error:", ok) + return ok + } + prcnum, ok := BConfig.Int("crontab::prcnum") + if ok != nil { + logs.Error("CreateIssue, config crontab::prcnum, error:", ok) + return ok + } + issueWhitelist, ok := BConfig.Int("cve::issue_whitelist") + if ok != nil { + logs.Error("CreateIssue, config cve::issue_whitelist, error: ", ok) + return ok + } + // Process lock status data + models.UpdateLockIssueStatus(15, 0) + beforeTime := common.GetBeforeTime(days) + if issueWhitelist == 1 { + limitErr := addLimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Info("CreateIssue, addLimitedIssue, err: ", limitErr) + logs.Info("The execution of the issue task of creating cve ends. . .") + return limitErr + } else { + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1) + logs.Info("CreateIssue, addUnlimitedIssue, err: ", unlimitErr) + logs.Info("The execution of the issue task of creating cve ends. . .") + return unlimitErr + } +} + +//ProcUpdateIssue Update issue +func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) error { + path := issueValue.PackName + // Query and modify score + sr, err := models.QueryIssueScoreRecord(issueValue.CveId, 0) + if err != nil { + logs.Error("ProcUpdateIssue, Failed to query score records, CveNum: ", + issueValue.CveNum, ", err: ", err) + } + if sr.NVDScore == 0 { + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("ProcUpdateIssue, Failed to get Score, err: ", + err, ", CveNum: ", issueValue.CveNum) + return errors.New("NVD评分缺失") + } + sr.NVDScore = sc.NVDScore + sr.NvectorVule = sc.NvectorVule + sr.CveId = issueValue.CveId + if sr.Id > 0 { + resultNum, upErr := models.UpdateScoreRecord(&sr, "NVDScore", "NvectorVule") + if upErr != nil { + logs.Error("UpdateScoreRecord, upErr: ", upErr, resultNum) + } + } else { + sr.Status = 0 + sr.CreateTime = time.Now() + scoreId, inErr := models.InsertScoreRecord(&sr) + logs.Error("InsertScoreRecord, upErr: ", inErr, scoreId) + } + } + if issueValue.OrganizationID == 3 { + // Get branch information + // Query the repo that needs to submit an issue + taskhandler.CreateBrandAndTags(accessToken, owner, path, issueValue.OrganizationID) + } + // Query issue template + var it models.IssueTemplate + it.CveNum = issueValue.CveNum + it.CveId = issueValue.CveId + templateErr := models.GetIssueTemplateByColName(&it, "CveNum", "CveId") + if templateErr == nil && it.TemplateId > 0 { + it.OwnedVersion = issueValue.CveVersion + it.Repo = issueValue.PackName + if len(issueValue.RepoName) > 0 { + it.OwnedComponent = issueValue.RepoName + } + if it.NVDScore == 0 && sr.NVDScore > 0 { + it.NVDScore = sr.NVDScore + it.NVDVector = sr.NvectorVule + } + it.CveBrief = issueValue.Description + it.CveLevel = issueValue.CveLevel + if (it.Assignee == "" || len(it.Assignee) < 2) && issueValue.OrganizationID == 1 { + repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) + if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { + assignee, assErr := taskhandler.GetCollaboratorInfo(accessToken, owner, path) + if assignee != "" && len(assignee) > 1 { + it.Assignee = assignee + //return err + } else { + logs.Error("ProcUpdateIssue, Get the repo: owner:", owner, + ", path:", path, ", Analyst failed, ", "assErr:", assErr, + ", CveNum: ", issueValue.CveNum, ", Create an issue without a maintainer") + } + } else { + it.Assignee = repoMainTainer + } + } + _, err := taskhandler.UpdateIssueToGit(accessToken, owner, path, issueValue, it) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("ProcUpdateIssue, Failed to update issue template, "+ + "CveNum: ", issueValue.CveNum, "err: ", err) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err + } + // Update issue status + models.UpdateIssueStatus(issueValue, 2) + // Update score status + models.UpdateIssueScore(issueValue, 2) + templetID, err := models.CreateIssueTemplate(&it) + if err != nil { + logs.Error("ProcUpdateIssue, Failed to modify issue template, "+ + "CveNum: ", issueValue.CveNum, ",err: ", err) + //return err + } + models.UpdateIssueScoreRe(issueValue, 1) + logs.Info("ProcUpdateIssue, Successfully updated the issue template, "+ + "CveNum: ", issueValue.CveNum, ", templetID: ", templetID) + // Judgment of necessary fields + if it.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("ProcUpdateIssue, Field is empty: NVDScore: ", it.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + if it.Status < 3 { + labelUnFix := beego.AppConfig.String("labelUnFix") + labelUnFixList := taskhandler.AddLabelValue(accessToken, path, it.IssueNum, owner, labelUnFix, 1) + if len(labelUnFixList) > 0 { + it.IssueLabel = strings.Join(labelUnFixList, ",") + updateBool := taskhandler.UpdateIssueLabels(accessToken, path, it.IssueNum, owner, it.IssueLabel) + if !updateBool { + logs.Error("UpdateIssueLabels, Update failed, IssueNum: ", it.IssueNum) + } + } + } + return errors.New("CVE的描述和NVD信息缺失") + } + } + return nil +} + +func EulerIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee string) error { + sn, err := models.QueryIssueSecurity(issueValue.CveId, issueValue.CveNum) + if err == nil && len(sn.CveNum) > 0 { + logs.Info("ProcIssue, QueryIssueSecurity, Query safety information: ", sn) + // Get issue handler + repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) + if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { + assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) + if assignee == "" { + logs.Error("ProcIssue, Get the repo： owner:", owner, ",path:", path, ",Analyst failed", + ", CveNum: ", issueValue.CveNum, ",Create an issue without a maintainer", ", err:", err) + } + } else { + assignee = repoMainTainer + } + } else { + logs.Error("ProcIssue, Failed to obtain security", ", CveNum: ", issueValue.CveNum, ", err: ", err) + return errors.New("CVE描述缺失") + } + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("ProcIssue, Failed to get Score, err: ", err, ", CveNum: ", issueValue.CveNum) + return errors.New("NVD评分缺失") + } + branchList := make([]string, 0) + errBrands := errors.New("") + // Get branch information + branchList, errBrands = taskhandler.GetBranchesInfo(accessToken, owner, path, issueValue.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("ProcIssue, Failed to obtain branch information,CveNum: ", issueValue.CveNum, + ", path: ", path, ", err: ", errBrands) + } + + legalFlag := false + if issueValue.DataSource == 4 { + if len(issueValue.CveNum) < 2 || len(issueValue.PackName) < 2 { + legalFlag = true + } + } else if issueValue.DataSource == 3 { + if len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } else { + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } + // Judgment of necessary fields + if legalFlag { + logs.Error("ProcIssue, Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("CVE的描述和NVD信息缺失") + } + resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, branchList) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("ProcIssue, Failed to create issue, err: ", err, ",resp: ", resp, ",CveNum: ", issueValue.CveNum) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err + } + return nil +} + +func GaussIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee string) error { + reviewerList, rlerr := models.GetGuassSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + assignee = v.NameSpace + break + } + } else { + logs.Error("GetGuassSecurityReviewerList, rlerr: ", rlerr) + } + branchList := make([]string, 0) + errBrands := errors.New("") + if len(issueValue.PackName) < 2 { + gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") + path = gaussIssuePath + } else { + path = issueValue.PackName + } + gaussBranchPath := beego.AppConfig.String("opengauss::gauss_branch_path") + // Get branch information + branchList, errBrands = taskhandler.GetBranchesInfo(accessToken, owner, gaussBranchPath, issueValue.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("ProcIssue, Failed to obtain branch information,CveNum: ", issueValue.CveNum, + ", path: ", path, ", err: ", errBrands) + gaussVersion := beego.AppConfig.String("opengauss::gauss_version") + if len(gaussVersion) > 0 { + gaussVerSlice := strings.Split(gaussVersion, ",") + if len(gaussVerSlice) > 0 { + branchList = append(branchList, gaussVerSlice...) + } + } + } + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("ProcIssue, Failed to get Score, err: ", err, ", cveId: ", issueValue.CveId) + return err + } + // Judgment of necessary fields + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("ProcIssue, Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("Field is empty") + } + resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, branchList) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("ProcIssue, Failed to create issue, err: ", err, + ",resp: ", resp, ",cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err + } + return nil +} + +func MindSporeIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee string) error { + reviewerList, rlerr := models.GetSporeSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + assignee = v.NameSpace + break + } + } else { + logs.Error("GetSporeSecurityReviewerList, rlerr: ", rlerr) + } + branchList := make([]string, 0) + // Get branch information + branchList = taskhandler.CreateBrandAndTags(accessToken, owner, path, issueValue.OrganizationID) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("ProcIssue, Failed to get Score, err: ", err, ", cveId: ", issueValue.CveId) + return err + } + // Judgment of necessary fields + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("ProcIssue, Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("Field is empty") + } + resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, branchList) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("ProcIssue, Failed to create issue, err: ", err, + ",resp: ", resp, ",cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err + } + return nil +} + +func OpenLooKengIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee string) error { + reviewerList, rlerr := models.GetOpenLookengSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + assignee = v.NameSpace + break + } + } else { + logs.Error("GetSporeSecurityReviewerList, rlerr: ", rlerr) + } + branchList := make([]string, 0) + // Get branch information + branchList = taskhandler.CreateBrandAndTags(accessToken, owner, path, issueValue.OrganizationID) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("ProcIssue, Failed to get Score, err: ", err, ", cveId: ", issueValue.CveId) + return err + } + // Judgment of necessary fields + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("ProcIssue, Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("Field is empty") + } + resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, branchList) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("ProcIssue, Failed to create issue, err: ", err, + ",resp: ", resp, ",cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err + } + return nil +} + +//ProcIssue Deal with issue +func ProcIssue(issueValue models.VulnCenter, accessToken, owner string) error { + assignee := "" + path := issueValue.PackName + if issueValue.OrganizationID == 1 { + eulerErr := EulerIssue(issueValue, accessToken, owner, path, assignee) + if eulerErr != nil { + logs.Error("EulerIssue, eulerErr: ", eulerErr, issueValue.CveNum) + return eulerErr + } + } else if issueValue.OrganizationID == 2 { + gaussErr := GaussIssue(issueValue, accessToken, owner, path, assignee) + if gaussErr != nil { + logs.Error("GaussIssue, gaussErr: ", gaussErr, issueValue.CveNum) + return gaussErr + } + } else if issueValue.OrganizationID == 3 { + sporeErr := MindSporeIssue(issueValue, accessToken, owner, path, assignee) + if sporeErr != nil { + logs.Error("MindSporeIssue, sporeErr: ", sporeErr, issueValue.CveNum) + return sporeErr + } + } else if issueValue.OrganizationID == 4 { + looKengErr := OpenLooKengIssue(issueValue, accessToken, owner, path, assignee) + if looKengErr != nil { + logs.Error("OpenLooKengIssue, looKengErr: ", looKengErr, issueValue.CveNum) + return looKengErr + } + } + return nil +} + +func EmgCreateIssue() error { + defer common.Catchs() + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("CreateIssue, config init, error:", err) + return err + } + years, ok := BConfig.Int("cve::cve_number") + if ok != nil { + logs.Error("CreateIssue, config cve::years, error:", ok) + return ok + } + manYears, ok := BConfig.Int("cve::cve_number_m") + if ok != nil { + logs.Error("CreateIssue, config cve::manYears, error:", ok) + return ok + } + toolYears, ok := BConfig.Int("cve::cve_number_t") + if ok != nil { + logs.Error("CreateIssue, config cve::toolYears, error:", ok) + return ok + } + prcnum, ok := BConfig.Int("crontab::prcnum") + if ok != nil { + logs.Error("CreateIssue, config crontab::prcnum, error:", ok) + return ok + } + issueWhitelist, ok := BConfig.Int("cve::issue_whitelist") + if ok != nil { + logs.Error("CreateIssue, config cve::issue_whitelist, error: ", ok) + return ok + } + beforeTime := time.Now().Add(-time.Minute * 60).Format("2006-01-02 15:04:05") + if len(beforeTime) > 19 { + beforeTime = beforeTime[:19] + } + if issueWhitelist == 1 { + limitErr := addLimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Info("CreateIssue, addLimitedIssue, err: ", limitErr) + logs.Info("The execution of the issue task of creating cve ends. . .") + return limitErr + } else { + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1) + logs.Info("CreateIssue, addUnlimitedIssue, err: ", unlimitErr) + logs.Info("The execution of the issue task of creating cve ends. . .") + return unlimitErr + } +} diff --git a/cve-vulner-manager/task/oricvecheck.go b/cve-vulner-manager/task/oricvecheck.go new file mode 100644 index 0000000000000000000000000000000000000000..ef9f8a247b603176404b91094d401b54360600c7 --- /dev/null +++ b/cve-vulner-manager/task/oricvecheck.go @@ -0,0 +1,109 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "strings" +) + +//CheckOriCve Check the CVE of the original data +func CheckOriCve() error { + defer common.Catchs() + logs.Info("The task of verifying the original cve data starts...") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("CheckOriCve, config init, error:", err) + return err + } + // The amount of data processed at a time + prcNum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("CheckOriCve, config crontab::prcnum, error: invalid value") + return errors.New("value is nil") + } + // Get the data source of the table + _, err = taskhandler.CheckCveOriginData(prcNum) + logs.Info("End of verifying original cve data task...") + return err +} + +// Unlock database table data +func UnLockTable() error { + defer common.Catchs() + logs.Info("Execute unlock table task start...") + beforeTime := common.GetBeforeHourTime(1) + // unlock center + models.UnLockUpdateIssueStatus(15, 0, common.GetCurTime(), beforeTime) + // unlock origin cve + models.UnLockUpdateCveIssueStatus(15, 0, common.GetCurTime(), beforeTime) + // unlock upstream + models.UnLockUpdateOriginStatus(15, 0, common.GetCurTime(), beforeTime) + // unlock excel + models.UnLockUpdateOriginExcelStatus(15, 0, common.GetCurTime(), beforeTime) + logs.Info("Execute unlock table End of task...") + return nil +} + +// Cve handles abnormal data reprocessing +func ProcAbnCve() error { + defer common.Catchs() + logs.Info("Correct the cve data and start the task...") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("ProcAbnCve, config init, error:", err) + return err + } + // The amount of data processed at a time + prcNum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("ProcAbnCve, config crontab::prcnum, error: invalid value") + return errors.New("value is nil") + } + days, err := BConfig.Int("crontab::days") + if err != nil { + logs.Error("ProcAbnCve, config crontab::days, error: invalid value") + return errors.New("value is nil") + } + cveStatuStr := BConfig.String("cve::abn_cve_status") + if cveStatuStr == "" { + logs.Error("ProcAbnCve, config cve::abn_cve_status error: invalid value") + return errors.New("value is nil") + } + st := strings.Split(cveStatuStr, ",") + for k, v := range st { + st[k] = "'" + v + "'" + } + cveSt := strings.Join(st, ",") + // Get the data source of the table + _, err = taskhandler.UpdateAbnCveStatus(prcNum, days, cveSt) + logs.Info("Correct cve data End of task...") + return err +} + +// Generate a certain number of SA numbers each time +func ProcSaNumber() error { + defer common.Catchs() + logs.Info("Generate SA number task start...") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("ProcSaNumber, config init, error:", err) + return err + } + // The amount of data processed at a time + saReAmount, err := BConfig.Int64("cve::sa_re_amount") + if err != nil { + logs.Error("ProcSaNumber, config cve::sa_re_amount, error: invalid value") + return errors.New("value is nil") + } + // Get the data source of the table + _, err = taskhandler.AddSaNumber(saReAmount) + logs.Info("Generate SA number task end...") + return err +} diff --git a/cve-vulner-manager/task/printlog.go b/cve-vulner-manager/task/printlog.go new file mode 100644 index 0000000000000000000000000000000000000000..e754a82a1bd449c6a4869ea70c54035a3a494d8b --- /dev/null +++ b/cve-vulner-manager/task/printlog.go @@ -0,0 +1,50 @@ +package task + +import ( + "errors" + "fmt" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" +) + +func DeletLogs() error { + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("DeletLogs, config init, error:", err) + return err + } + logDir := BConfig.String("log::log_dir") + if logDir == "" { + logs.Error("DeletLogs, config excel::log_dir, error: invalid value") + return errors.New("value is nil") + } + fileInfoList, err := ioutil.ReadDir(logDir) + if err != nil { + logs.Error(err) + return err + } + fmt.Println(len(fileInfoList)) + for i := range fileInfoList { + if fileInfoList[i].Name() != "cve.log" { + err := os.Remove(logDir + "/" + fileInfoList[i].Name()) + if err != nil { + logs.Error("Failed to delete file: ", fileInfoList[i].Name()) + } + } + } + return nil +} + +// ProcLogData Processing log +func ProcLogData() error { + logs.Info("Create log task starts") + // delete logs + err := DeletLogs() + if err != nil { + logs.Error(err) + } + logs.Info("End of log creation task") + return err +} diff --git a/cve-vulner-manager/task/synccve.go b/cve-vulner-manager/task/synccve.go new file mode 100644 index 0000000000000000000000000000000000000000..d70b10111a3130587b0dcaae9b00f86e447fa19b --- /dev/null +++ b/cve-vulner-manager/task/synccve.go @@ -0,0 +1,110 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" +) + +// Sync data to cve vulnerability and create issue +func SyncCveAndIssue() error { + syncErr := SyncCveData() + if syncErr != nil { + logs.Error("SyncCveAndIssue, SyncCveData, err: ", syncErr) + } + issueErr := SyncCreateIssue() + if issueErr != nil { + logs.Error("SyncCveAndIssue, SyncCreateIssue, err: ", issueErr) + return issueErr + } + return nil +} + +func SyncCveData() error { + defer common.Catchs() + logs.Info("Start the task of generating cve raw data into cve library.") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init, error: ", err) + return err + } + // The amount of data processed at a time + prcNum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("crontab::prcnum, error: ", err) + return errors.New("value is nil") + } + // How many days have been processed to date data + days := -1 + // openeuler Number start value + cveRef := BConfig.String("cve::cveref") + openeulerNum, oErr := BConfig.Int("cve::openeulernum") + if oErr != nil { + logs.Error("cve::openeulernum, err:", err) + return oErr + } + // cve credibility level + credibilityLevel, cErr := BConfig.Int("cve::credibility_level") + if cErr != nil { + logs.Error("cve::credibility_level, err:", err) + return cErr + } + // Process raw data obtained by api + ok, syncErr := taskhandler.GetCveOriginData(prcNum, days, openeulerNum, credibilityLevel, cveRef) + if !ok || syncErr != nil { + logs.Error("ProcCveOriginData, GetCveOriginData, err: ", err) + } + return syncErr +} + +func SyncCreateIssue() error { + defer common.Catchs() + logs.Info("The issue task of creating cve begins. . .") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("CreateIssue, config init, error:", err) + return err + } + days := -1 + years, ok := BConfig.Int("cve::cve_number") + if ok != nil { + logs.Error("CreateIssue, config cve::years, error:", ok) + return ok + } + manYears, ok := BConfig.Int("cve::cve_number_m") + if ok != nil { + logs.Error("CreateIssue, config cve::manYears, error:", ok) + return ok + } + toolYears, ok := BConfig.Int("cve::cve_number_t") + if ok != nil { + logs.Error("CreateIssue, config cve::toolYears, error:", ok) + return ok + } + prcnum, ok := BConfig.Int("crontab::prcnum") + if ok != nil { + logs.Error("CreateIssue, config crontab::prcnum, error:", ok) + return ok + } + issueWhitelist, ok := BConfig.Int("cve::issue_whitelist") + if ok != nil { + logs.Error("CreateIssue, config cve::issue_whitelist, error: ", ok) + return ok + } + beforeTime := common.GetBeforeTime(days) + if issueWhitelist == 1 { + limitErr := addLimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Info("CreateIssue, addLimitedIssue, err: ", limitErr) + logs.Info("The execution of the issue task of creating cve ends. . .") + return limitErr + } else { + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1) + logs.Info("CreateIssue, addUnlimitedIssue, err: ", unlimitErr) + logs.Info("The execution of the issue task of creating cve ends. . .") + return unlimitErr + } +} diff --git a/cve-vulner-manager/task/urgent.go b/cve-vulner-manager/task/urgent.go new file mode 100644 index 0000000000000000000000000000000000000000..7c731c54744ca14560b496b16bf34262c54444a5 --- /dev/null +++ b/cve-vulner-manager/task/urgent.go @@ -0,0 +1,56 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" +) + +// Synchronize excel data +func SynchronizeExcelData() error { + defer common.Catchs() + logs.Info("Start the task of generating cve raw data into cve library.") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("SynchronizeExcelData, config init, error: ", err) + return err + } + // The amount of data processed at a time + prcNum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("SynchronizeExcelData, crontab::prcnum, error: ", err) + return errors.New("value is nil") + } + // How many days have been processed to date data + days, ok := BConfig.Int("crontab::days") + if ok != nil { + logs.Error("SynchronizeExcelData, crontab::days, err:", err) + return ok + } + // openeuler Number start value + cveRef := BConfig.String("cve::cveref") + openeulerNum, ok := BConfig.Int("cve::openeulernum") + if ok != nil { + logs.Error("SynchronizeExcelData, cve::openeulernum, err:", err) + return ok + } + // Processing excel raw data + okx, err := taskhandler.GetCveOriginExcelData(prcNum, days, openeulerNum, cveRef) + if !okx { + logs.Error("SynchronizeExcelData, GetCveOriginExcelData, err: ", err) + return err + } + return nil +} + +func HandUrgentData() error{ + // Synchronize excel data + syErr := SynchronizeExcelData() + // Create cve data + cErr := SyncCreateIssue() + logs.Info(syErr, cErr) + return nil +} diff --git a/cve-vulner-manager/task/yaml.go b/cve-vulner-manager/task/yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..0bd70f1a0b4738191799efa1310b6f85c9bfb359 --- /dev/null +++ b/cve-vulner-manager/task/yaml.go @@ -0,0 +1,58 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" +) + +// GetYamlData Get yaml data source +func GetYamlData() error { + defer common.Catchs() + logs.Info("Get the yaml data source task started") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("GetYamlData, config init, error:", err) + return err + } + apiUrl := BConfig.String("yaml::apiurl") + if apiUrl == "" { + logs.Error("GetYamlData, config yaml::apiurl, error: invalid value") + return errors.New("value is nil") + } + // Get the data source of the table + _, err = taskhandler.GetYamlTables(apiUrl) + // Get yaml + if err == nil { + _, err = taskhandler.GetYamlByGit(apiUrl) + } + // Synchronize other sources of yaml version + taskhandler.SyncEulerYaml() + // Delete historical yaml source version data that does not exist + taskhandler.DelHistoryEulerYaml() + logs.Info("End of the task of obtaining yaml data source") + return err +} + +// GetEulerYamlData Get yaml data source +func GetEulerYamlData() error { + defer common.Catchs() + logs.Info("Get the euleryaml data source task started") + // Query the cve to be processed, 1: add; 2: modify + eulerUrl := beego.AppConfig.String("yaml::eulerurl") + if eulerUrl == "" { + logs.Error("GetEulerYamlData, config yaml::eulerUrl, error: invalid value") + return errors.New("eulerUrl value is nil") + } + _, err := taskhandler.GetEulerYamlInfo(eulerUrl) + // Synchronize other sources of yaml version + taskhandler.SyncEulerYaml() + // Delete historical yaml source version data that does not exist + taskhandler.DelHistoryEulerYaml() + logs.Info("End of the task of obtaining euleryaml data source") + return err +} diff --git a/cve-vulner-manager/taskhandler/assist.go b/cve-vulner-manager/taskhandler/assist.go new file mode 100644 index 0000000000000000000000000000000000000000..95b71afe5f07605f0bb45d55057011a0f778c2f8 --- /dev/null +++ b/cve-vulner-manager/taskhandler/assist.go @@ -0,0 +1,227 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/util" + "encoding/json" + "errors" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "os" + "sort" + "strings" +) + +func GetOautToken(gt GiteeToken) { + url := "https://gitee.com/oauth/token" + var req util.RequestInfo + req.URL = url + req.Data = make(map[string]string) + req.Data["grant_type"] = gt.GrantType + req.Data["username"] = gt.UserName + req.Data["password"] = gt.Password + req.Data["client_id"] = gt.ClientID + req.Data["client_secret"] = gt.ClientSecret + req.Data["scope"] = gt.Scope + resp, err := util.PostURLEncoded(req) + if err != nil { + logs.Error("Failed to obtain token,url: ", url, + ",Request parameter: ", gt, ", err:", err) + return + } + var respDict map[string]interface{} + err = json.Unmarshal(resp, &respDict) + if err != nil { + logs.Error(err) + return + } + if _, ok := respDict["access_token"]; !ok { + logs.Error("Failed to obtain token, err: ", ok, ", url: ", url) + return + } + GitToken := respDict["access_token"].(string) + os.Setenv("GITEE_TOKEN", GitToken) +} + +func GetCollaboratorInfo(accessToken string, owner string, path string) (string, error) { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/collaborators?access_token=" + accessToken + collabor, err := util.HTTPGet(url) + if err == nil && collabor != nil { + for _, value := range collabor { + if _, ok := value["id"]; !ok { + logs.Error("collaborators, err: ", ok, "url: ", url) + continue + } + assignee := "" + flag := false + for mapKey, mapValue := range value { + if mapKey == "login" { + assignee = mapValue.(string) + } + if mapKey == "permissions" { + mapValuex := mapValue.(map[string]interface{}) + for perKey, perValue := range mapValuex { + if perKey == "admin" && perValue.(bool) == true { + flag = true + } + } + } + } + if assignee != "" && flag { + return assignee, nil + } + } + } else { + logs.Error("Failed to obtain repo owner, owner:", owner, ",path:", path, ",err: ", err) + return "", err + } + } + return "", errors.New("参数错误") +} + +func GetBranchesInfo(accessToken, owner, path string, organizationID int8) ([]string, error) { + BrandVerSlice := make([]string, 0) + if organizationID == 4 { + openlookengVersion := beego.AppConfig.String("openlookeng::openlookeng_version") + if len(openlookengVersion) > 0 { + BrandVerSlice = strings.Split(openlookengVersion, ",") + } + } else if organizationID == 3 { + mindsporeVersion := beego.AppConfig.String("mindspore::mindspore_version") + if len(mindsporeVersion) > 0 { + BrandVerSlice = strings.Split(mindsporeVersion, ",") + } + } else if organizationID == 2 { + gaussVersion := beego.AppConfig.String("opengauss::gauss_version") + if len(gaussVersion) > 0 { + BrandVerSlice = strings.Split(gaussVersion, ",") + } + path = beego.AppConfig.String("opengauss::openGauss-server") + } else { + eulerVersion := beego.AppConfig.String("cve::affected_branchs") + if len(eulerVersion) > 0 { + BrandVerSlice = strings.Split(eulerVersion, ",") + } + } + branchNameList := []string{} + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/branches?access_token=" + accessToken + branch, err := util.HTTPGet(url) + if err == nil && branch != nil { + for _, value := range branch { + if _, ok := value["name"]; !ok { + logs.Error("branches, err: ", ok, "url: ", url) + continue + } + mapValue := value["name"].(string) + if mapValue != "" && len(mapValue) > 1 { + if organizationID == 2 { + if !common.IsContainsLetters(mapValue) { + branchNameList = append(branchNameList, mapValue) + } + } else if organizationID == 3 { + if common.IsContainSpecialChar(mapValue, 1) { + branchNameList = append(branchNameList, mapValue) + } + } else if organizationID == 4 { + if !common.IsContainsLetters(mapValue) { + branchNameList = append(branchNameList, mapValue) + } + } else { + for _, gv := range BrandVerSlice { + if gv == mapValue { + branchNameList = append(branchNameList, mapValue) + } + } + } + } + } + } else { + logs.Error("Failed to get branch name, "+ + "err: ", err, ",owner: ", owner, ",path: ", path) + if organizationID != 1 { + logs.Info("1.Obtained branch information: ", BrandVerSlice) + return BrandVerSlice, err + } + logs.Info("2.Obtained branch information: ", branchNameList) + return branchNameList, err + } + } + if len(branchNameList) > 0 { + sort.Strings(branchNameList) + } + if organizationID == 2 { + if len(branchNameList) > 2 { + branchNameList = branchNameList[len(branchNameList)-2:] + } + } else if organizationID == 3 { + if len(branchNameList) > 4 { + branchNameList = branchNameList[len(branchNameList)-4:] + } + } else if organizationID == 4 { + if len(branchNameList) > 1 { + branchNameList = branchNameList[len(branchNameList)-1:] + } + } + if len(BrandVerSlice) > 0 { + branchNameList = append(branchNameList, BrandVerSlice...) + } + repList := make([]string, 0) + if len(branchNameList) > 0 { + sort.Strings(branchNameList) + tmpList := common.SliceRemoveDup(branchNameList) + for _, tl := range tmpList { + repList = append(repList, tl.(string)) + } + branchNameList = repList + } + logs.Info("3.Obtained branch information: ", branchNameList) + return branchNameList, nil +} + +func GetTagsInfo(accessToken, owner, path string, organizationID int8) ([]string, error) { + tagsList := []string{} + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/tags?access_token=" + accessToken + branch, err := util.HTTPGet(url) + if err == nil && branch != nil { + for _, value := range branch { + if _, ok := value["name"]; !ok { + logs.Error("tagsInfo, err: ", ok, "url: ", url) + continue + } + mapValue := value["name"].(string) + if mapValue != "" && len(mapValue) > 1 { + if organizationID == 4 { + if !common.IsContainsLetters(mapValue) { + tagsList = append(tagsList, mapValue) + } + } else { + if common.IsContainSpecialChar(mapValue, 2) { + tagsList = append(tagsList, mapValue) + } + } + } + } + } else { + logs.Error("Failed to get tags name, "+ + "err: ", err, ",owner: ", owner, ",path: ", path) + return tagsList, err + } + } + if len(tagsList) > 0 { + sort.Strings(tagsList) + } + if organizationID == 3 { + if len(tagsList) > 4 { + tagsList = tagsList[len(tagsList)-4:] + } + } else if organizationID == 4 { + if len(tagsList) > 1 { + tagsList = tagsList[len(tagsList)-1:] + } + } + logs.Info("GetTagsInfo, tagsList:", tagsList) + return tagsList, nil +} diff --git a/cve-vulner-manager/taskhandler/comment.go b/cve-vulner-manager/taskhandler/comment.go new file mode 100644 index 0000000000000000000000000000000000000000..83f40166df8b2dc8d974e4707377656234551511 --- /dev/null +++ b/cve-vulner-manager/taskhandler/comment.go @@ -0,0 +1,29 @@ +package taskhandler + +import ( + "cvevulner/util" + "fmt" + "github.com/astaxie/beego/logs" +) + +//AddCommentToIssue Add a comment to the issue +func AddCommentToIssue(msg, issueNum, owner, repo, access string) { + url := fmt.Sprintf(`https://gitee.com/api/v5/repos/%v/%v/issues/%v/comments`, owner, repo, issueNum) + param := fmt.Sprintf(`{"access_token": "%s","body":"%s"}`, access, msg) + res, err := util.HTTPPost(url, param) + if err != nil { + logs.Error(err, msg) + } + logs.Info("Add comment back:", res, msg) +} + +//SendPrivateLetters Send a private message to a gitee user +func SendPrivateLetters(access, content, useName string) { + url := "https://gitee.com/api/v5/notifications/messages" + param := fmt.Sprintf(`{"access_token":"%s","username":"%s","content":"%s"}`, access, useName, content) + res, err := util.HTTPPost(url, param) + if err != nil { + logs.Error(err) + } + logs.Info("Send private message:", res) +} diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go new file mode 100644 index 0000000000000000000000000000000000000000..72bd468c8e872ac4c2bda1f2d7fc9e2bfb5a5236 --- /dev/null +++ b/cve-vulner-manager/taskhandler/common.go @@ -0,0 +1,1275 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/base64" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "os" + "strconv" + "strings" +) + +const ( + //GiteOrgInfoURL get gitee org info url + GiteOrgInfoURL = `https://gitee.com/api/v5/orgs/%v?access_token=%v` + //GiteOrgReposURL get all repository url + GiteOrgReposURL = `https://gitee.com/api/v5/orgs/%v/repos?access_token=%v&type=all&page=%v&per_page=%v` + //GiteRepoIssuesURL get issue list url + GiteRepoIssuesURL = `https://gitee.com/api/v5/repos/%v/%v/issues?access_token=%v&state=%v&sort=created&direction=desc&page=%v&per_page=%v` + //GiteRepoBranch get repo branch url + GiteRepoBranch = `https://gitee.com/api/v5/repos/%v/%v/branches?access_token=%v` + //RepoInfoURL get repo info url + RepoInfoURL = "https://api.openeuler.org/pkgmanage/packages/packageInfo?table_name=openEuler_LTS_20.03&pkg_name=%s" + perPage = 50 + //IssueType Types of issues crawled + CIssueType = "CVE和安全问题" + BranchRep = `(\(.*\))|(（.*）)` +) + +type GiteeToken struct { + GrantType string + UserName string + Password string + ClientID string + ClientSecret string + Scope string +} + +const bodyTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openEuler评分： + %v + 受影响版本排查(受影响/不受影响)： + %v + 修复是否涉及abi变化(是/否)： + %v +` + +const bodyUpTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openEuler评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v + 修复是否涉及abi变化(是/否)： + %v +` +const bodySecLinkTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openEuler评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v + 修复是否涉及abi变化(是/否)： + %v +三、漏洞修复 +安全公告链接：%v +` + +const commentCopyValue = ` +%v +**issue处理注意事项:** +**1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容,未引入的分支不需要填写, 否则无法关闭当前issue;** +**3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openEuler评分, 受影响版本排查(受影响/不受影响), 修复是否涉及abi变化(是/否))不能省略,省略后cve-manager将无法正常解析填写内容.** +************************************************************************ +影响性分析说明: + + +openEuler评分: (评分和向量) + + +受影响版本排查(受影响/不受影响): +%v +修复是否涉及abi变化(是/否)： +%v +----------------------------------------------------------------------- +issue处理具体操作请参考: +%v +pr关联issue具体操作请参考: +%v +` + +const gaussBodyTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openGauss评分： + %v + 受影响版本排查(受影响/不受影响)： + %v +` + +const SporeBodyTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + MindSpore评分： + %v + 受影响版本排查(受影响/不受影响)： + %v +` + +const gaussBodyUpTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openGauss评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v +` + +const SporeBodyUpTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + MindSpore评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v +` +const LooKengBodyTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openLooKeng评分： + %v + 受影响版本排查(受影响/不受影响)： + %v +` +const LooKengBodyUpTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openLooKeng评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v +` + +const gaussCommentCopyValue = ` +%v +**issue处理注意事项:** +**1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容,未引入的分支不需要填写, 否则无法关闭当前issue;** +**3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openGauss评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** +************************************************************************ +影响性分析说明: + + +openGauss评分: (评分和向量) + + +受影响版本排查(受影响/不受影响): +%v +----------------------------------------------------------------------- +issue处理具体操作请参考: +%v +pr关联issue具体操作请参考: +%v +` + +const SporeCommentCopyValue = ` +%v +**issue处理注意事项:** +**1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容,未引入的分支不需要填写, 否则无法关闭当前issue;** +**3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, MindSpore评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** +************************************************************************ +影响性分析说明: + + +MindSpore评分: (评分和向量) + + +受影响版本排查(受影响/不受影响): +%v +----------------------------------------------------------------------- +issue处理具体操作请参考: +%v +pr关联issue具体操作请参考: +%v +` +const LooKengCommentCopyValue = ` +%v +**issue处理注意事项:** +**1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容,未引入的分支不需要填写, 否则无法关闭当前issue;** +**3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openLooKeng评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** +************************************************************************ +影响性分析说明: + + +openLooKeng评分: (评分和向量) + + +受影响版本排查(受影响/不受影响): +%v +----------------------------------------------------------------------- +issue处理具体操作请参考: +%v +pr关联issue具体操作请参考: +%v +` + +const PrIssueLink = "https://gitee.com/help/articles/4142" + +// Create first comment +func CommentTemplate(assignee, commentCmd, affectedVersion, path string, assignLoginList []string) string { + maintainerList, mainOk := models.QueryRepoAllMaintainer(path) + assList := []string{} + assigneeStr := "" + if len(assignLoginList) > 0 { + for _, v := range assignLoginList { + assList = append(assList, "@"+v+" ") + } + } else { + if mainOk && len(maintainerList) > 0 { + for _, v := range maintainerList { + assList = append(assList, "@"+v.MemberName+" ") + } + } + } + if len(assList) > 0 { + assigneeStr = strings.Join(assList, ",") + } else { + assigneeStr = "@" + assignee + } + commentTemplate := fmt.Sprintf(commentCopyValue, assigneeStr, affectedVersion, affectedVersion, commentCmd, PrIssueLink) + return commentTemplate +} + +func SporeCommentTemplate(assignee, commentCmd, affectedVersion string) string { + anName := []string{} + reviewerList, rlerr := models.GetSporeSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + anName = append(anName, "@"+v.NameSpace+" ") + } + } else { + logs.Error("GetSporeSecurityReviewerList, rlerr: ", rlerr) + } + if len(anName) > 1 { + assignee = strings.Join(anName, ",") + } else { + assignee = "@" + assignee + " " + } + commentTemplate := fmt.Sprintf(SporeCommentCopyValue, assignee, affectedVersion, commentCmd, PrIssueLink) + return commentTemplate +} + +func GaussCommentTemplate(assignee, commentCmd, affectedVersion string) string { + anName := []string{} + reviewerList, rlerr := models.GetGuassSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + anName = append(anName, "@"+v.NameSpace+" ") + } + } else { + logs.Error("GetGuassSecurityReviewerList, rlerr: ", rlerr) + } + if len(anName) > 1 { + assignee = strings.Join(anName, ",") + } else { + assignee = "@" + assignee + " " + } + commentTemplate := fmt.Sprintf(gaussCommentCopyValue, assignee, affectedVersion, commentCmd, PrIssueLink) + return commentTemplate +} + +func LooKengCommentTemplate(assignee, commentCmd, affectedVersion string) string { + anName := []string{} + reviewerList, rlerr := models.GetOpenLookengSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + anName = append(anName, "@"+v.NameSpace+" ") + } + } else { + logs.Error("GetOpenLookengSecurityReviewerList, rlerr: ", rlerr) + } + if len(anName) > 1 { + assignee = strings.Join(anName, ",") + } else { + assignee = "@" + assignee + " " + } + commentTemplate := fmt.Sprintf(LooKengCommentCopyValue, assignee, affectedVersion, commentCmd, PrIssueLink) + return commentTemplate +} + +func GitOpenEulerData(values map[string]interface{}, ge *models.GitOpenEuler, tb models.GitPackageTable) { + defer common.Catchs() + ge.TableName = tb.TableName + ge.TableId = tb.TableId + ge.Status = 1 + if values["feature"] == nil { + ge.Feature = "" + } else { + switch values["feature"].(type) { + case string: + ge.Feature = values["feature"].(string) + case int: + ge.Feature = strconv.Itoa(values["feature"].(int)) + case int64: + ge.Feature = strconv.FormatInt(values["feature"].(int64), 10) + case float64: + ge.Feature = strconv.FormatInt(int64(values["feature"].(float64)), 10) + default: + ge.Feature = "" + } + } + if values["url"] == nil { + ge.OriginUrl = "" + } else { + ge.OriginUrl = values["url"].(string) + } + switch values["id"].(type) { + case string: + ge.PackageId, _ = strconv.ParseInt(values["id"].(string), 10, 64) + case int: + ge.PackageId = values["id"].(int64) + case int64: + ge.PackageId = values["id"].(int64) + case float64: + ge.PackageId = int64(values["id"].(float64)) + default: + ge.PackageId = 0 + } + if values["name"] == nil { + ge.PackageName = "" + } else { + packName := values["name"].(string) + if packName != "" { + packName = common.DeletePreAndSufSpace(packName) + } + ge.PackageName = packName + } + if values["version"] == nil { + ge.Version = "" + } else { + version := values["version"].(string) + if version != "" { + version = common.DeletePreAndSufSpace(version) + } + ge.Version = version + } + if values["release"] == nil { + ge.Release = "" + } else { + release := values["release"].(string) + if release != "" { + release = common.DeletePreAndSufSpace(release) + } + ge.Release = release + } + if values["rpm_license"] == nil { + ge.License = "" + } else { + ge.License = values["rpm_license"].(string) + } + if values["maintainer"] == nil { + ge.MainTainer = "" + } else { + ge.MainTainer = values["maintainer"].(string) + } + if values["maintainlevel"] == nil { + ge.MainTainLevel = 0 + } else { + ge.MainTainLevel = values["maintainlevel"].(int8) + } + if values["release_time"] == nil { + ge.ReleaseTime = "" + } else { + ge.ReleaseTime = values["release_time"].(string) + } + switch values["used_time"].(type) { + case string: + ge.UsedTime = values["used_time"].(string) + case int: + ge.UsedTime = strconv.Itoa(values["used_time"].(int)) + case int64: + ge.UsedTime = strconv.FormatInt(values["used_time"].(int64), 10) + case float64: + ge.UsedTime = strconv.FormatInt(int64(values["used_time"].(float64)), 10) + default: + ge.UsedTime = "" + } + if values["latest_version"] == nil { + ge.LatestVersion = "" + } else { + ge.LatestVersion = values["latest_version"].(string) + } + if values["latest_version_time"] == nil { + ge.LatestVersionTime = "" + } else { + ge.LatestVersionTime = values["latest_version_time"].(string) + } + switch values["issue"].(type) { + case string: + ge.IssueCount, _ = strconv.ParseInt(values["issue"].(string), 10, 64) + case int: + ge.IssueCount = values["issue"].(int64) + case int64: + ge.IssueCount = values["issue"].(int64) + case float64: + ge.IssueCount = int64(values["issue"].(float64)) + default: + ge.IssueCount = 0 + } +} + +func GitOpenEulerInfoData(values map[string]interface{}, gp *models.GitPackageInfo, ge models.GitOpenEuler) { + defer common.Catchs() + gp.GitId = ge.GitId + gp.Ids = 0 + if values["pkg_name"] == nil { + gp.PackageName = "" + } else { + PackageName := values["pkg_name"].(string) + if PackageName != "" { + PackageName = common.DeletePreAndSufSpace(PackageName) + } + gp.PackageName = PackageName + } + if values["version"] == nil { + gp.Version = "" + } else { + Version := values["version"].(string) + if Version != "" { + Version = common.DeletePreAndSufSpace(Version) + } + gp.Version = Version + } + if values["release"] == nil { + gp.Release = "" + } else { + Release := values["release"].(string) + if Release != "" { + Release = common.DeletePreAndSufSpace(Release) + } + gp.Release = Release + } + if values["url"] == nil { + gp.OriginUrl = "" + } else { + gp.OriginUrl = values["url"].(string) + } + if values["license"] == nil { + gp.License = "" + } else { + gp.License = values["license"].(string) + } + switch values["feature"].(type) { + case string: + gp.Feature = values["feature"].(string) + case int: + gp.Feature = strconv.Itoa(values["feature"].(int)) + case int64: + gp.Feature = strconv.FormatInt(values["feature"].(int64), 10) + case float64: + gp.Feature = strconv.FormatInt(int64(values["feature"].(float64)), 10) + default: + gp.Feature = "" + } + if values["maintainer"] == nil { + gp.MainTainer = "" + } else { + gp.MainTainer = values["maintainer"].(string) + } + if values["maintainlevel"] == nil { + gp.MainTainLevel = 0 + } else { + gp.MainTainLevel = values["maintainlevel"].(int8) + } + if values["gitee_url"] == nil { + gp.GitUrl = "" + } else { + gp.GitUrl = values["gitee_url"].(string) + } + if values["summary"] == nil { + gp.Summary = "" + } else { + gp.Summary = values["summary"].(string) + } + if values["description"] == nil { + gp.Decription = "" + } else { + gp.Decription = values["description"].(string) + } + BuildRequired := "" + if values["buildrequired"].([]interface{}) != nil && len(values["buildrequired"].([]interface{})) > 0 { + for _, vx := range values["buildrequired"].([]interface{}) { + BuildRequired = BuildRequired + vx.(string) + "," + } + gp.BuildRequired = BuildRequired[:len(BuildRequired)-1] + } else { + gp.BuildRequired = BuildRequired + } + gp.Status = 0 +} + +type GitTablePackCount struct { + Page int + Size int + TableName string + Count int64 +} + +func OrgRepoParams(path, brand string) string { + index := -1 + if strings.Index(brand, ":") != -1 { + index = strings.Index(brand, ":") + } else if strings.Index(brand, "：") != -1 { + index = strings.Index(brand, "：") + } + tmpBrandx := brand + brandx := brand + if index != -1 { + brandx = brand[:index] + } + ori := models.QueryEulerOriginBRepo(path, brandx) + if len(ori) > 0 { + for _, or := range ori { + if len(or.Branchs) > 0 { + brSlice := strings.Split(or.Branchs, ",") + if len(brSlice) > 0 { + for _, br := range brSlice { + if br == brandx { + if len(or.Version) > 0 && len(or.Version) < 32 { + if index != -1 { + return brand[:index] + "(" + or.Version + ")" + brand[index:] + } + return brandx + "(" + or.Version + ")" + } + } + } + } + } + } + } + return tmpBrandx +} + +func AffectVersionExtract(brandArray []string, itsAffectedVersion, packName string, organizationID int8) string { + affectedVersion := "" + if itsAffectedVersion != "" && len(itsAffectedVersion) > 0 { + brandsGroup := strings.Split(itsAffectedVersion, ",") + if len(brandsGroup) > 0 { + for i, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brandx := "" + if organizationID == 1 { + brandx = common.BranchVersionRep(brand) + brandx = OrgRepoParams(packName, brandx) + } else { + brandx = brand + } + //logs.Info("brand1: ", brandx) + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brandx + "\n" + } + } + } else { + if len(brandArray) > 0 { + for i, brand := range brandArray { + if brand == "" || len(brand) < 2 { + continue + } + brandx := "" + if organizationID == 1 { + brandx = common.BranchVersionRep(brand) + brandx = OrgRepoParams(packName, brandx) + } else { + brandx = brand + } + logs.Info("brand2: ", brandx) + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brandx + ":\n" + } + } else { + affectedVersion = "\n" + } + } + logs.Info("affectedVersion: ", affectedVersion) + return affectedVersion +} + +func updateTemplateAbi(brandArray []string, its models.IssueTemplate) { + cols := make([]string, 0) + if len(its.AbiVersion) < 2 && len(brandArray) > 0 { + abiVersionStr := "" + for _, abi := range brandArray { + abiVersionStr = abiVersionStr + abi + ":," + } + if len(abiVersionStr) > 2 { + its.AbiVersion = abiVersionStr[:len(abiVersionStr)-1] + cols = append(cols, "AbiVersion") + } + } + if len(its.AffectedVersion) < 2 && len(brandArray) > 0 { + AffectedVersionStr := "" + for _, ver := range brandArray { + AffectedVersionStr = AffectedVersionStr + ver + ":," + } + if len(AffectedVersionStr) > 2 { + its.AffectedVersion = AffectedVersionStr[:len(AffectedVersionStr)-1] + cols = append(cols, "AffectedVersion") + } + } + if len(cols) > 0 { + err := models.UpdateIssueTemplate(&its, cols...) + if err != nil { + logs.Error("updateTemplateAbi, UpdateIssueTemplate, err: ", updateTemplateAbi) + } + } +} + +func AddLabelValue(accessToken, path, issueNum, owner, issueLabel string, flag int8) []string { + tmpLabelSlice := make([]string, 0) + if len(issueNum) > 1 { + if flag == 1 { + tmpLabelSlice, _ = QueryIssueLabels(accessToken, path, issueNum, owner) + } else { + _, tmpLabelSlice = QueryIssueLabels(accessToken, path, issueNum, owner) + } + } + if len(issueLabel) > 1 { + issueLabel := strings.Split(issueLabel, ",") + for _, lab := range issueLabel { + if len(tmpLabelSlice) > 0 { + labFlag := false + for _, labs := range tmpLabelSlice { + if strings.ToUpper(labs) == strings.ToUpper(lab) { + labFlag = true + break + } + } + if !labFlag { + tmpLabelSlice = append(tmpLabelSlice, lab) + } + } else { + tmpLabelSlice = append(tmpLabelSlice, lab) + } + } + } + labelSlice := make([]string, 0) + tmpList := common.SliceRemoveDup(tmpLabelSlice) + for _, tl := range tmpList { + labelSlice = append(labelSlice, tl.(string)) + } + return labelSlice +} + +func CreateIssueBody(accessToken, owner, path, assignee string, + cve models.VulnCenter, sc models.Score, openEulerScore, score, labels string, + its models.IssueTemplate, flag int, issueType, pkgLink string, brandArray []string) string { + requestBody := "" + scoreType := "" + if sc.ScoreType == "v2" { + scoreType = "2.0" + } else { + scoreType = "3.0" + } + if len(brandArray) == 0 { + if cve.OrganizationID == 1 || cve.OrganizationID == 2 { + brandArray, _ = GetBranchesInfo(accessToken, owner, path, cve.OrganizationID) + } else if cve.OrganizationID == 3 || cve.OrganizationID == 4 { + brandArray = CreateBrandAndTags(accessToken, owner, path, cve.OrganizationID) + } + } + if cve.Description != "" && len(cve.Description) > 1 { + cve.Description = strings.Replace(cve.Description, "\n", "", -1) + cve.Description = strings.ReplaceAll(cve.Description, "\"", " ") + cve.Description = strings.ReplaceAll(cve.Description, "'", " ") + cve.Description = strings.ReplaceAll(cve.Description, "\\", " ") + } + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return "" + } + commentCmd := BConfig.String("reflink::comment_cmd") + gaussCommentCmd := BConfig.String("reflink::gauss_comment_cmd") + sporeCommentCmd := BConfig.String("reflink::spore_comment_cmd") + looKengCommentCmd := BConfig.String("reflink::looKeng_comment_cmd") + floatOpenEulerScore, err := strconv.ParseFloat(openEulerScore, 64) + if err != nil { + floatOpenEulerScore = 0.0 + } + nvdScore, err := strconv.ParseFloat(score, 64) + if err != nil { + nvdScore = 0.0 + } + cveAnalysis := "" + if its.CveAnalysis != "" && len(its.CveAnalysis) > 1 { + cveAnalysis = strings.Replace(its.CveAnalysis, "\n", "", -1) + cveAnalysis = strings.ReplaceAll(cveAnalysis, "'", " ") + cveAnalysis = strings.ReplaceAll(cveAnalysis, "\"", " ") + cveAnalysis = strings.ReplaceAll(cveAnalysis, "\\", " ") + } + affectedVersion := AffectVersionExtract(brandArray, its.AffectedVersion, cve.PackName, cve.OrganizationID) + abiVersion := AffectVersionExtract(brandArray, its.AbiVersion, cve.PackName, cve.OrganizationID) + updateTemplateAbi(brandArray, its) + logs.Info("its.CreateTime: ", its.CreateTime) + updateTime := its.CreateTime.Format(common.DATE_FORMAT) + if len(updateTime) >= 19 && its.TemplateId > 0 { + updateTime = common.TimeConverStr(updateTime[:19]) + } else { + updateTime = common.GetLocalCurTime() + } + logs.Info("cve.UpdateTime.String(): ", updateTime, its.CreateTime, cve.CreateTime) + if updateTime != "" && len(updateTime) > 19 { + updateTime = updateTime[:19] + } + StatusName := "" + if its.StatusName != "" && len(its.StatusName) > 1 { + if its.StatusName == "待办的" || its.StatusName == "开启的" || + strings.ToLower(its.StatusName) == "open" { + StatusName = "open" + } else if its.StatusName == "进行中" || strings.ToLower(its.StatusName) == "started" || + strings.ToLower(its.StatusName) == "progressing" { + StatusName = "progressing" + } else if its.StatusName == "已完成" || strings.ToLower(its.StatusName) == "closed" { + StatusName = "closed" + if AffectVersion(its.AffectedVersion) == 2 { + labels = beego.AppConfig.String("labelFixed") + } else if AffectVersion(its.AffectedVersion) == 3 { + labels = beego.AppConfig.String("labeUnaffected") + } + } else if its.StatusName == "已拒绝" || strings.ToLower(its.StatusName) == "rejected" { + StatusName = "rejected" + return "" + } else if its.StatusName == "已挂起" || strings.ToLower(its.StatusName) == "suspended" { + StatusName = "suspended" + return "" + } else { + StatusName = its.StatusName + return "" + } + } + labelSlice := make([]string, 0) + labelSlice = AddLabelValue(accessToken, path, its.IssueNum, owner, labels, 1) + //labelSlice = append(labelSlice, labels) + labels = strings.Join(labelSlice, ",") + its.IssueLabel = labels + tpErr := models.UpdateIssueTemplate(&its, "IssueLabel") + if tpErr != nil { + logs.Error(tpErr) + } + bodyUpTplx := bodyUpTpl + bodyTplx := bodyTpl + if cve.OrganizationID == 2 { + bodyUpTplx = gaussBodyUpTpl + bodyTplx = gaussBodyTpl + commentCmd = gaussCommentCmd + } else if cve.OrganizationID == 3 { + bodyUpTplx = SporeBodyUpTpl + bodyTplx = SporeBodyTpl + commentCmd = sporeCommentCmd + } else if cve.OrganizationID == 4 { + bodyUpTplx = LooKengBodyUpTpl + bodyTplx = LooKengBodyTpl + commentCmd = looKengCommentCmd + } + cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" + cve.CveDetailUrl = "https://nvd.nist.gov/vuln/detail/" + cve.CveNum + cve.CveLevel = models.OpenEulerScoreProc(nvdScore) + if len(cve.CveVersion) > 0 && cve.CveVersion[0] == ',' { + cve.CveVersion = cve.CveVersion[1:] + } + if flag == 1 { + if floatOpenEulerScore > 0.0 || (its.OpenEulerVector != "" && len(its.OpenEulerVector) > 1) { + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + sc.NvectorVule + oVector := scoreType + "/" + its.OpenEulerVector + body := "" + if its.Status == 3 && len(its.SecLink) > 3 && cve.OrganizationID == 1 { + body = fmt.Sprintf(bodySecLinkTpl, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion, abiVersion, its.SecLink) + } else { + if cve.OrganizationID == 1 { + body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion, abiVersion) + } else { + body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion) + } + } + requestBody = fmt.Sprintf(`{ + "access_token": "%s", + "repo": "%s", + "title": "%s", + "state": "%s", + "body": "%s", + "assignee": "%s", + "labels": "%s", + "security_hole": "false" + }`, accessToken, path, cve.CveNum, StatusName, body, assignee, labels) + } else { + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + sc.NvectorVule + openEulerScore = " " + body := "" + if cve.OrganizationID == 1 { + body = fmt.Sprintf(bodyTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion, abiVersion) + } else { + body = fmt.Sprintf(bodyTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion) + } + requestBody = fmt.Sprintf(`{ + "access_token": "%s", + "repo": "%s", + "title": "%s", + "state": "%s", + "body": "%s", + "assignee": "%s", + "labels": "%s", + "security_hole": "false" + }`, accessToken, path, cve.CveNum, StatusName, body, assignee, labels) + } + } else if flag == 2 { + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + sc.NvectorVule + openEulerScore = " " + body := "" + if cve.OrganizationID == 1 { + body = fmt.Sprintf(bodyTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion, abiVersion) + } else { + body = fmt.Sprintf(bodyTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion) + } + requestBody = fmt.Sprintf(`{ + "access_token": "%s", + "repo": "%s", + "title": "%s", + "issue_type": "%s", + "body": "%s", + "assignee": "%s", + "labels": "%s", + "security_hole": "false" + }`, accessToken, path, cve.CveNum, issueType, body, assignee, labels) + } else { + if floatOpenEulerScore > 0.0 || (its.OpenEulerVector != "" && len(its.OpenEulerVector) > 1) { + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + its.NVDVector + oVector := scoreType + "/" + its.OpenEulerVector + body := "" + if its.Status == 3 && len(its.SecLink) > 3 && cve.OrganizationID == 1 { + body = fmt.Sprintf(bodySecLinkTpl, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion, abiVersion, its.SecLink) + } else { + if cve.OrganizationID == 1 { + body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion, abiVersion) + } else { + body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion) + } + } + requestBody = fmt.Sprintf(`{ + "access_token": "%s", + "repo": "%s", + "title": "%s", + "state": "%s", + "body": "%s", + "assignee": "%s", + "labels": "%s", + "security_hole": "false" + }`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) + } else { + openEulerScore = " " + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + sc.NvectorVule + body := "" + if cve.OrganizationID == 1 { + body = fmt.Sprintf(bodyTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion, abiVersion) + } else { + body = fmt.Sprintf(bodyTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion) + } + requestBody = fmt.Sprintf(`{ + "access_token": "%s", + "repo": "%s", + "title": "%s", + "state": "%s", + "body": "%s", + "assignee": "%s", + "labels": "%s", + "security_hole": "false" + }`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) + } + } + return requestBody +} + +func AffectVersion(affectedVersion string) int { + affectBranchsxList := make([]string, 0) + affectValue := make([]string, 0) + unaffectValue := make([]string, 0) + unFixValue := make([]string, 0) + flag := 1 + affectedBranchs := beego.AppConfig.String("cve::affected_branchs") + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + if affectedVersion != "" && len(affectedVersion) > 1 { + affectProductList := strings.Split(affectedVersion, ",") + if len(affectProductList) > 0 { + for _, brands := range affectProductList { + branchSlice := []string{} + if strings.Contains(brands, ":") { + branchSlice = strings.Split(brands, ":") + } else if strings.Contains(brands, "：") { + branchSlice = strings.Split(brands, "：") + } + if len(affectBranchsxList) > 0 { + for _, arr := range affectBranchsxList { + if len(branchSlice) == 2 { + affBrand := common.BranchVersionRep(branchSlice[0]) + if strings.TrimSpace(affBrand) == arr { + if len(strings.TrimSpace(branchSlice[1])) > 1 { + if strings.TrimSpace(branchSlice[1]) == "受影响" { + affectValue = append(affectValue, arr) + } else { + unaffectValue = append(unaffectValue, arr) + } + } else { + unFixValue = append(unFixValue, arr) + } + } + } else { + if len(branchSlice) > 0 { + affBrand := common.BranchVersionRep(branchSlice[0]) + if strings.TrimSpace(affBrand) == arr { + unFixValue = append(unFixValue, arr) + } + } + } + } + } + } + } + } + if len(affectValue) > 0 && len(unFixValue) == 0 { + flag = 2 + } else if len(unaffectValue) > 0 && len(affectValue) == 0 && len(unFixValue) == 0 { + flag = 3 + } + return flag +} + +func RemoveSubstring(s string, subList []string) string { + newStr := s + if len(subList) > 0 { + for _, sub := range subList { + if strings.Contains(newStr, sub) { + if sub == "

" { + newStr = strings.ReplaceAll(newStr, sub, "") + } else { + newStr = strings.ReplaceAll(newStr, sub, " ") + } + + } + } + } + newStr = strings.TrimSpace(newStr) + return newStr +} + +func deleteTailBlank(str string) string { + spaceNum := 0 + for i := len(str) - 1; i >= 0; i-- { + if str[i] == ' ' { + spaceNum++ + } else { + break + } + } + return str[:len(str)-spaceNum] +} + +func IssueStateConversion(status int8) string { + var retName string + switch status { + case 2: + retName = "进行中" + default: + retName = "待办的" + } + return retName +} + +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func baseStdEncode(srcBtye []byte) string { + encoding := base64.StdEncoding.EncodeToString(srcBtye) + return encoding +} + +func XmlSpecCharHand(xmlStr string) string { + tempXmlStr := strings.ReplaceAll(xmlStr, "&", "&") + tempXmlStr = strings.ReplaceAll(tempXmlStr, "<", "<") + tempXmlStr = strings.ReplaceAll(tempXmlStr, ">", ">") + tempXmlStr = strings.ReplaceAll(tempXmlStr, "'", "'") + tempXmlStr = strings.ReplaceAll(tempXmlStr, "\"", """) + return tempXmlStr +} + +func GetOrganizationId(nameSpace string) (organizationID int8) { + communityName := beego.AppConfig.String("communityName") + if len(communityName) > 1 { + communityList := strings.Split(communityName, ",") + for _, cl := range communityList { + cli := strings.Split(cl, ":") + if len(cli) > 1 && nameSpace == cli[0] { + id, ok := strconv.Atoi(cli[1]) + if ok == nil { + organizationID = int8(id) + } + } + } + } + return +} + +// Query label +func QueryIssueLabels(token, repo, issueNum, owner string) ([]string, []string) { + allLabelSlice := make([]string, 0) + labelSlice := make([]string, 0) + totalLabel := beego.AppConfig.String("totallabel") + totalLabelList := strings.Split(totalLabel, ",") + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/issues/%v/labels?access_token=%v", owner, repo, issueNum, token) + labelData, err := util.HTTPGet(url) + if err == nil && labelData != nil { + for _, value := range labelData { + if _, ok := value["id"]; !ok { + logs.Error("labelData, err: ", ok, "url: ", url) + continue + } + labelStr := value["name"].(string) + allLabelSlice = append(allLabelSlice, labelStr) + labFlag := false + for _, lab := range totalLabelList { + if strings.ToUpper(labelStr) == strings.ToUpper(lab) { + labFlag = true + break + } + } + if !labFlag { + labelSlice = append(labelSlice, labelStr) + } + } + } + return labelSlice, allLabelSlice +} + +// Add, change, delete tags +func UpdateIssueLabels(token, repo, issueNum, owner, label string) bool { + labelStr := label + labelSlice := strings.Split(label, ",") + if len(labelSlice) > 0 { + laSlice := []string{} + for _, la := range labelSlice { + laSlice = append(laSlice, fmt.Sprintf("\"%v\"", la)) + } + if len(laSlice) > 0 { + labelStr = strings.Join(laSlice, ",") + } + } + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/issues/%v/labels?access_token=%v", owner, repo, issueNum, token) + reqBody := fmt.Sprintf("[%v]", labelStr) + logs.Info("UpdateIssueLabels, reqBody: ", reqBody) + resp, err := util.HTTPPut(url, reqBody) + if err != nil { + logs.Error("UpdateIssueLabels, Failed to update label, url: ", url, ", err: ", err) + return false + } + if resp == nil || len(resp) == 0 { + return true + } + if _, ok := resp[0]["id"]; !ok { + logs.Error("UpdateIssueLabels, Failed to update label, err: ", ok, ", url: ", url) + return false + } + return true +} + +func RegMatchCve(str string) string { + reg := util.RegMatchCve + if reg == nil { + fmt.Println("RegMatchCve err") + return "" + } + reslist := reg.FindAllString(str, -1) + if len(reslist) > 0 { + return reslist[0] + } else { + return "" + } +} diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go new file mode 100644 index 0000000000000000000000000000000000000000..239cc0ad900103cc98c996e0f3308c60d5a1a5bd --- /dev/null +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -0,0 +1,927 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/json" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +var updateLock sync.Mutex + +func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, + sc models.Score, resp map[string]interface{}, + path, assignee, issueType, labels, owner string) { + issueTemp.CveId = cve.CveId + issueTemp.CveNum = cve.CveNum + issueTemp.OwnedComponent = cve.RepoName + issueTemp.OwnedVersion = cve.CveVersion + if issueTemp.TemplateId == 0 { + issueTemp.MtAuditFlag = 1 + issueTemp.SaAuditFlag = 0 + } + issueTemp.NVDScore = sc.NVDScore + issueTemp.NVDVector = sc.NvectorVule + issueTemp.CveBrief = cve.Description + issueTemp.CveLevel = cve.CveLevel + if resp != nil && len(resp) > 0 { + issueTemp.IssueId = int64(resp["id"].(float64)) + issueTemp.IssueNum = resp["number"].(string) + issueTemp.StatusName = resp["state"].(string) + if strings.ToLower(resp["state"].(string)) == "open" || + resp["state"].(string) == "待办的" || + resp["state"].(string) == "开启的" { + issueTemp.Status = 1 + issueTemp.StatusName = "open" + issueTemp.IssueStatus = 1 + } else if strings.ToLower(resp["state"].(string)) == "started" || + strings.ToLower(resp["state"].(string)) == "progressing" || + strings.ToLower(resp["state"].(string)) == "进行中" { + issueTemp.Status = 2 + issueTemp.StatusName = "progressing" + issueTemp.IssueStatus = 3 + } else if strings.ToLower(resp["state"].(string)) == "closed" || resp["state"].(string) == "已完成" { + issueTemp.Status = 3 + issueTemp.StatusName = "closed" + issueTemp.IssueStatus = 2 + } else { + if strings.ToLower(resp["state"].(string)) == "rejected" || resp["state"].(string) == "已拒绝" { + issueTemp.StatusName = "rejected" + issueTemp.Status = 4 + } + if strings.ToLower(resp["state"].(string)) == "suspended" || resp["state"].(string) == "已挂起" { + issueTemp.StatusName = "suspended" + issueTemp.Status = 5 + } + issueTemp.IssueStatus = 6 + } + } else { + issueTemp.IssueId = 0 + issueTemp.IssueNum = "nil" + issueTemp.StatusName = "nil" + issueTemp.Status = 0 + } + issueTemp.Assignee = assignee + issueTemp.IssueLabel = labels + issueTemp.Owner = owner + issueTemp.Repo = cve.PackName + issueTemp.Title = cve.CveNum + issueTemp.IssueType = issueType + issueTemp.Collaborators = "" + issueTemp.Milestone = "" + issueTemp.Program = "" + issueTemp.SecurityHole = 0 + logs.Info("CreateIssueData, Assemble issue, store template data in db: ", issueTemp) +} + +func GetGiteeIssue(accessToken, owner, path, issueNum string) (error, map[string]interface{}) { + var respBody map[string]interface{} + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/issues/%v?access_token=%v", owner, path, issueNum, accessToken) + if accessToken != "" && owner != "" && path != "" && issueNum != "" { + issueInfo, err := util.HTTPGetCom(url) + if err == nil && issueInfo != nil { + err = json.Unmarshal(issueInfo, &respBody) + if err != nil { + logs.Error(err) + return err, respBody + } + if respBody != nil && respBody["number"] != nil && respBody["number"].(string) == issueNum { + return nil, respBody + } + } else { + logs.Error("GetIssueNum, http request failed, url: ", url) + return err, respBody + } + } + return errors.New("error"), respBody +} + +func GetRepoMember(accessToken, owner, path string) (error, []string, []string) { + var respBody []map[string]interface{} + assignLoginList := []string{} + assignNameList := []string{} + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/collaborators?access_token=%v&page=1&per_page=20", owner, path, accessToken) + if accessToken != "" && owner != "" && path != "" { + issueInfo, err := util.HTTPGetCom(url) + if err == nil && issueInfo != nil { + err = json.Unmarshal(issueInfo, &respBody) + if err != nil { + logs.Error(err) + return err, assignLoginList, assignNameList + } + if len(respBody) > 0 { + for _, rb := range respBody { + if value, ok := rb["id"]; ok { + if int(value.(float64)) > 0 { + if rb["login"].(string) != "openeuler-ci-bot" { + assignLoginList = append(assignLoginList, rb["login"].(string)) + assignNameList = append(assignNameList, rb["name"].(string)) + } + } + } + } + return nil, assignLoginList, assignNameList + } + } else { + logs.Error("GetRepoMember, http request failed, url: ", url) + return err, assignLoginList, assignNameList + } + } + return errors.New("error"), assignLoginList, assignNameList +} + +func OPenCheckWhetherIssue(cveNumber, repoPath, owner, accessToken string, organizationID int8) bool { + vc := models.VulnCenter{CveNum: cveNumber, PackName: repoPath, OrganizationID: organizationID} + vcErr := models.GetVulnCenterByCid(&vc, "CveNum", "PackName", "OrganizationID") + if vcErr != nil { + logs.Error("GetVulnCenterByCid, vcErr: ", vcErr, ",cveNum: ", cveNumber) + return false + } + issueTmp := models.IssueTemplate{} + issueTmp.CveNum = cveNumber + issueTmp.Repo = repoPath + issueTmp.CveId = vc.CveId + _ = models.GetIssueTemplateByColName(&issueTmp, "CveNum", "Repo", "CveId") + if issueTmp.TemplateId > 0 && len(issueTmp.IssueNum) > 1 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, repoPath, issueTmp.IssueNum) + if issueErr == nil { + logs.Info("OPenCheckWhetherIssue, Issue has been created,issueBody:", issueBody) + return true + } + } + return false +} + +func CreateIssueToGit(accessToken, owner, path, assignee string, + cve models.VulnCenter, sc models.Score, brandArray []string) (string, error) { + defer common.Catchs() + var it models.IssueTemplate + it.CveId = cve.CveId + it.CveNum = cve.CveNum + _, assignLoginList, _ := GetRepoMember(accessToken, owner, path) + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if cve.OrganizationID == 1 { + sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + it.Assignee = sia.Assignee + assignee = sia.Assignee + } + } + assigneeGite := "" + if cve.DataSource == 4 && cve.CveId == it.CveId && len(it.IssueNum) > 2 { + logs.Info("Current data is synchronized") + } else { + if it.TemplateId > 0 && len(it.IssueNum) > 2 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, it.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(it.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } else { + if issueBody != nil && issueBody["assignee"] != nil { + assigneeObj := issueBody["assignee"].(map[string]interface{}) + if assigneeObj != nil && assigneeObj["login"] != nil { + assigneeGite = assigneeObj["login"].(string) + } + } + } + } else { + logs.Error("CreateIssueToGit, GetIssueTemplateByColName, templateErr: ", templateErr, ",it: ", it) + } + } + + if it.TemplateId > 0 && len(it.IssueNum) > 2 { + if it.Assignee == "" || len(it.Assignee) == 0 { + it.Assignee = assignee + } + issueType := it.IssueType + labels := "" + if it.IssueLabel != "" && len(it.IssueLabel) > 1 { + labels = it.IssueLabel + } else { + labels = beego.AppConfig.String("labelUnFix") + } + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + it.IssueNum + score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) + OpenEulerScore := strconv.FormatFloat(it.OpenEulerScore, 'f', 1, 64) + if len(it.AffectedVersion) > 1 { + brandArryTmp := strings.Split(it.AffectedVersion, ",") + for _, brand := range brandArray { + if !strings.Contains(it.AffectedVersion, brand+":") && + !strings.Contains(it.AffectedVersion, brand+"：") && + it.Status != 3 { + brandArryTmp = append(brandArryTmp, brand+":") + } + } + brandStr := strings.Join(brandArryTmp, ",") + it.AffectedVersion = brandStr + } else { + if len(brandArray) > 0 { + var brandArryTmp []string + for _, brand := range brandArray { + brandArryTmp = append(brandArryTmp, brand+":") + } + brandStr := strings.Join(brandArryTmp, ",") + it.AffectedVersion = brandStr + } + } + if len(assigneeGite) > 1 { + it.Assignee = assigneeGite + } else { + if cve.OrganizationID == 1 && len(assignLoginList) > 0 { + it.Assignee = assignLoginList[0] + } + } + requestBody := CreateIssueBody(accessToken, owner, path, it.Assignee, + cve, sc, OpenEulerScore, score, labels, it, 1, it.IssueType, "", brandArray) + if requestBody != "" && len(requestBody) > 1 { + logs.Info("CreateIssueToGit, Template data before issue creation: ", requestBody) + updateLock.Lock() + resp, err := util.HTTPPatch(url, requestBody) + updateLock.Unlock() + if err != nil { + logs.Error("CreateIssueToGit, Update issue failed, cveNum: ", + cve.CveNum, ",err: ", err, ",url: ", url) + return "", errors.New("调用gitee更新issue接口失败") + } + if _, ok := resp["id"]; !ok { + logs.Error("CreateIssueToGit, Update issue failed, err: ", ok, ", url: ", url) + return "", errors.New("调用gitee更新issue接口失败") + } + logs.Info("CreateIssueToGit, Update issue successfully, "+ + "cveNum: ", cve.CveNum, "issueNum: ", resp["number"].(string)) + // Structure data + //var issueTemp models.IssueTemplate + CreateIssueData(&it, cve, sc, resp, path, it.Assignee, issueType, labels, owner) + // Store issue data + issTempID, err := models.CreateIssueTemplate(&it) + if err != nil { + logs.Error("CreateIssueToGit, Failed to update data of issue template,"+ + " cveNum: ", cve.CveNum, ",err: ", err) + //return "", nil + } + logs.Info("CreateIssueToGit, Successfully updated the data of the issue template, "+ + "issTempID: ", issTempID, ",cveNum: ", cve.CveNum) + } else { + logs.Info("CreateIssueToGit, No need to update the issue template and issue status, its: ", it) + } + // Update issue status + models.UpdateIssueStatus(cve, 2) + // Update score status + models.UpdateIssueScore(cve, 2) + // Update score record status + models.UpdateIssueScoreRe(cve, 1) + } + } else { + // Determine whether an issue has been created + if cve.OrganizationID == 1 { + gi := models.QueryGiteOriginIssue(cve.CveNum, cve.PackName, cve.OrganizationID) + if len(gi) > 0 { + for _, g := range gi { + issueErr, _ := GetGiteeIssue(accessToken, owner, path, g.Number) + if issueErr == nil { + models.DeleteIssueTemplate(it.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } + } + } + if len(assignLoginList) > 0 { + assignee = assignLoginList[0] + } + } + if it.TemplateId > 0 { + models.DeleteIssueTemplate(it.TemplateId) + } + issueType := CIssueType + labels := beego.AppConfig.String("labelUnFix") + if accessToken != "" && owner != "" && path != "" { + var issueTemp models.IssueTemplate + issueTemp.CreateTime = time.Now() + CreateIssueData(&issueTemp, cve, sc, nil, path, assignee, issueType, labels, owner) + // Store issue data + issTempID, templateErr := models.InsertIssueTemplate(&issueTemp) + if templateErr == nil && issTempID > 0 { + url := "https://gitee.com/api/v5/repos/" + owner + "/issues" + score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) + requestBody := CreateIssueBody(accessToken, owner, path, assignee, + cve, sc, "", score, labels, it, 2, issueType, "", brandArray) + logs.Info("CreateIssueToGit, isssue_body: ", requestBody) + if requestBody != "" && len(requestBody) > 1 { + updateLock.Lock() + resp, err := util.HTTPPost(url, requestBody) + updateLock.Unlock() + if err != nil { + logs.Error("CreateIssueToGit, url: ", url, ", Failed to create issue, "+ + "cveNum: ", cve.CveNum, ", err: ", err, ",url:", url) + models.DeleteIssueTemplate(issTempID) + return "", errors.New("调用gitee的创建issue接口失败") + } + if _, ok := resp["id"]; !ok { + logs.Error("CreateIssueToGit, Failed to create issue, "+ + "err: ", ok, ", url: ", url) + models.DeleteIssueTemplate(issTempID) + return "", errors.New("调用gitee的创建issue接口失败") + } + //var issueTemps models.IssueTemplate + issueTemp.TemplateId = issTempID + CreateIssueData(&issueTemp, cve, sc, resp, path, assignee, issueType, labels, owner) + if len(brandArray) > 0 { + var brandArrayTmp []string + for _, brand := range brandArray { + brandArrayTmp = append(brandArrayTmp, brand+":") + } + brandStr := strings.Join(brandArrayTmp, ",") + issueTemp.AffectedVersion = brandStr + issueTemp.AbiVersion = brandStr + issueTemp.SaAuditFlag = 0 + } else { + issueTemp.SaAuditFlag = 1 + } + // Store issue data + issTempIDx, idxErr := models.UpdateIssueTemplateAll(&issueTemp) + if idxErr != nil { + logs.Error("CreateIssueToGit, Failed to create data for issue template, "+ + "cveNum: ", cve, ",err: ", err) + //models.DeleteIssueTemplate(issTempID) + return "", errors.New("内部错误, 模板存储失败") + } + logs.Info("CreateIssueToGit, The data of the issue template was successfully, "+ + "created, issTempID: ", issTempIDx, "cveNum: ", cve.CveNum) + // Create issue comment + affectedVersion := "" + if len(brandArray) > 0 { + for i, brand := range brandArray { + if brand == "" || len(brand) < 2 { + continue + } + brandx := "" + if cve.OrganizationID == 1 { + brandx = common.BranchVersionRep(brand) + brandx = OrgRepoParams(cve.PackName, brandx) + } else { + brandx = brand + } + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brandx + ":\n" + } + } else { + affectedVersion = affectedVersion + "\n" + } + issueNum := resp["number"].(string) + errx := CreateIssueComment(accessToken, owner, path, assignee, cve, issueNum, affectedVersion, assignLoginList) + logs.Info("CreateIssueToGit, Issue comment creation result, err: ", errx) + if cve.OrganizationID == 1 { + // Trigger third-party data generation + agencyUrl := beego.AppConfig.String("cveagency::url") + agencyReq := make(map[string]interface{}, 0) + agencyReq["CveNum"] = issueTemp.CveNum + agencyReq["PackName"] = issueTemp.OwnedComponent + agencyReq["PackVersion"] = issueTemp.OwnedVersion + if len(brandArray) > 0 { + agencyReq["AffectBranch"] = strings.Join(brandArray, " ") + } else { + agencyReq["AffectBranch"] = "" + } + agencyReq["IssueNumber"] = issueTemp.IssueNum + util.HTTPPostCom(agencyReq, agencyUrl) + } + // Send gitee private message + content := fmt.Sprintf("%v 仓库有新的CVE和安全问题的ISSUE被 cve-manager 创建,CVE编号: %v, 请及时处理.", path, cve.CveNum) + SendPrivateLetters(accessToken, content, assignee) + //issueNum := resp["number"].(string) + issueID := int64(resp["id"].(float64)) + err = CreateDepositHooks(accessToken, owner, path, cve, issueNum, issueID) + if err != nil { + logs.Error("CreateIssueToGit, Failed to create hooks, cveNum: ", cve.CveNum, "err: ", err) + return "", errors.New("创建webhook失败") + } + logs.Info("CreateIssueToGit, Create hooks successfully, cveNum: ", cve.CveNum) + // Update issue status + models.UpdateIssueStatus(cve, 2) + // Update score status + models.UpdateIssueScore(cve, 2) + // Update score record status + models.UpdateIssueScoreRe(cve, 1) + } else { + models.DeleteIssueTemplate(issTempID) + } + } else { + logs.Error("CreateIssueToGit, Repeat issue, cve: ", cve, + ", templateErr: ", templateErr) + return "", nil + } + } + } + branchs := "" + if len(brandArray) > 0 { + for _, b := range brandArray { + if b != "" && len(b) > 1 { + branchs = branchs + b + "/" + } + } + if branchs != "" && len(branchs) > 1 { + branchs = branchs[:len(branchs)-1] + } + } + // Store security bulletin related information + var sec models.SecurityNotice + CreateSecNoticeData(&sec, cve, path, branchs, sc.NVDScore) + secID, noticeErr := models.UpdateSecNotice(&sec) + if noticeErr != nil { + logs.Error("CreateIssueToGit, Failed to update security information,"+ + "CveNum: ", cve.CveNum, "path: ", path, "err: ", noticeErr) + return "", nil + } + logs.Info("CreateIssueToGit, Update security information successfully, "+ + "secID: ", secID, "cveNum: ", cve.CveNum) + return "", nil +} + +func UpdateIssueToGit(accessToken, owner, path string, + cve models.VulnCenter, its models.IssueTemplate) (string, error) { + logs.Info("UpdateIssueToGit, Update template request parameters: cve: ", cve, + ",its: ", its, ", owner: ", owner, ",path: ", path) + if its.Status == 4 || its.Status == 5 { + logs.Error("UpdateIssueToGit, "+ + "The current issue has been suspended/rejected and will not be processed, its: ", its) + models.UpdateIssueStatus(cve, 2) + return "", errors.New("The current issue has been suspended/rejected and will not be processed") + } + owner, accessToken = common.GetOwnerAndToken(cve.OrganizationID) + if cve.OrganizationID == 2 { + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } + } + assigneeGite := "" + if cve.DataSource == 4 && cve.CveId == its.CveId && len(its.IssueNum) > 2 { + logs.Info("Current data is synchronized") + } else { + if its.IssueNum != "" && len(its.IssueNum) > 2 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, its.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } else { + if issueBody != nil { + if issueBody != nil && issueBody["assignee"] != nil { + assigneeObj := issueBody["assignee"].(map[string]interface{}) + if assigneeObj != nil && assigneeObj["login"] != nil { + assigneeGite = assigneeObj["login"].(string) + } + } + } + } + } else { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } + } + //labels := its.IssueLabel + if cve.OrganizationID == 1 { + sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + its.Assignee = sia.Assignee + } + } + labels := "" + if its.IssueLabel != "" && len(its.IssueLabel) > 1 { + labels = its.IssueLabel + } else { + labels = beego.AppConfig.String("labelUnFix") + } + pkgList, err := models.QueryPackageByCveId(its.CveId) + pkgLink := "" + var brandArray []string + if err == nil && len(pkgList) > 0 { + for _, p := range pkgList { + pkgLink = pkgLink + fmt.Sprintf(`[%v](%v)\r\n`, p.PackName, p.PackUrl) + } + } + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + its.IssueNum + score := strconv.FormatFloat(its.NVDScore, 'f', 1, 64) + OpenEulerScore := strconv.FormatFloat(its.OpenEulerScore, 'f', 1, 64) + var sc models.Score + sc, scok := models.QueryScoreByCveId(cve.CveId) + if !scok { + logs.Error("UpdateIssueToGit, Score does not exist, cve: ", cve, cve.CveNum) + } + if len(assigneeGite) > 1 { + its.Assignee = assigneeGite + } + requestBody := CreateIssueBody(accessToken, owner, path, its.Assignee, + cve, sc, OpenEulerScore, score, labels, its, 3, its.IssueType, pkgLink, brandArray) + logs.Info("UpdateIssueToGit, isssue_body: ", requestBody) + if requestBody != "" && len(requestBody) > 1 { + updateLock.Lock() + resp, err := util.HTTPPatch(url, requestBody) + updateLock.Unlock() + if err != nil { + logs.Error("UpdateIssueToGit, Update issue failed, cveNum: ", cve.CveNum, "err: ", err) + return "", errors.New("调用gitee更新issue的接口失败") + } + if _, ok := resp["id"]; !ok { + logs.Error("UpdateIssueToGit, Failed to create issue, err: ", ok, "url: ", url, cve.CveNum) + return "", errors.New("调用gitee更新issue的接口失败") + } + // Store security bulletin related information + var sec models.SecurityNotice + CreateSecNoticeData(&sec, cve, path, its.AffectedVersion, its.OpenEulerScore) + secId, err := models.UpdateSecNotice(&sec) + if err != nil { + logs.Error("UpdateIssueToGit, Failed to update security information, "+ + "CveNum: ", cve.CveNum, ",path: ", path, ",err: ", err) + return "", errors.New("内部错误, 数据错误") + } else { + logs.Info("UpdateIssueToGit, Update security information successfully, "+ + "secId: ", secId, ",cveNum: ", cve.CveNum) + } + } + } + return "", nil +} + +func CreateIssueHookData(issHook *models.IssueHooks, resp map[string]interface{}, + path, owner string, issueNum string, issueId, cveId int64) *models.IssueHooks { + issHook.CveId = cveId + issHook.IssueId = issueId + issHook.IssueNum = issueNum + issHook.HookId = int64(resp["id"].(float64)) + issHook.Owner = owner + issHook.Repo = path + issHook.CreateTime = common.GetCurTime() + issHook.UpdateTime = common.GetCurTime() + issHook.Status = 1 + issHook.HookUrl = resp["url"].(string) + if resp["push_events"].(bool) == true { + issHook.PushEvent = 1 + } else { + issHook.PushEvent = 0 + } + if resp["tag_push_events"].(bool) == true { + issHook.TagPushEvent = 1 + } else { + issHook.TagPushEvent = 0 + } + if resp["issues_events"].(bool) == true { + issHook.IssueEvent = 1 + } else { + issHook.IssueEvent = 0 + } + if resp["note_events"].(bool) == true { + issHook.NoteEvent = 1 + } else { + issHook.NoteEvent = 0 + } + if resp["merge_requests_events"].(bool) == true { + issHook.MergeRequestEvent = 1 + } else { + issHook.MergeRequestEvent = 0 + } + return issHook +} + +func PatchWebHooks(accessToken, owner, path, hookurl, pwd, issueNum, + hookId string, issueId, cveId int64) error { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks/" + hookId + push_events := "true" + tag_push_events := "true" + issues_events := "true" + note_events := "true" + merge_requests_events := "true" + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "url": "%s", + "password": "%s", + "push_events": "%s", + "tag_push_events": "%s", + "issues_events": "%s", + "note_events": "%s", + "merge_requests_events": "%s" + }`, accessToken, hookurl, pwd, push_events, + tag_push_events, issues_events, note_events, merge_requests_events) + logs.Info("PatchWebHooks, hook_body: ", requestBody) + resp, err := util.HTTPPatch(url, requestBody) + if err != nil { + logs.Error("PatchWebHooks, Failed to update webhook, url: ", url, ", err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("PatchWebHooks, Failed to update webhook, err: ", ok, ", url: ", url) + return errors.New("更新仓库 hook失败") + } + if resp["password"].(string) == pwd { + var issHook models.IssueHooks + CreateIssueHookData(&issHook, resp, + path, owner, issueNum, issueId, cveId) + hookId, err := models.CreateDepositHooks(&issHook) + if err != nil { + logs.Error("PatchWebHooks, Failed to update webhook, repo: ", path, ", err: ", err) + return nil + } else { + logs.Info("PatchWebHooks, Update webhook successfully, hookId: ", hookId, "repo: ", path) + } + } + return nil + } + return errors.New("更新仓库 hook失败") +} + +func PostWebHooks(accessToken, owner, path, hookurl, pwd, issueNum string, issueId, cveId int64) error { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks" + push_events := "true" + tag_push_events := "true" + issues_events := "true" + note_events := "true" + merge_requests_events := "true" + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "url": "%s", + "password": "%s", + "push_events": "%s", + "tag_push_events": "%s", + "issues_events": "%s", + "note_events": "%s", + "merge_requests_events": "%s" + }`, accessToken, hookurl, pwd, push_events, + tag_push_events, issues_events, note_events, merge_requests_events) + logs.Info("hook_body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("PatchWebHooks, Failed to create webhook, url: ", url, + ", cveId:", cveId, ",err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("PatchWebHooks, Failed to create webhook, err: ", ok, "url: ", url) + return errors.New("创建仓库hook失败") + } + if resp["password"].(string) == pwd { + var issHook models.IssueHooks + CreateIssueHookData(&issHook, resp, + path, owner, issueNum, issueId, cveId) + // Store issue data + hookId, err := models.CreateDepositHooks(&issHook) + if err != nil { + logs.Error("PatchWebHooks, Failed to create webhook, path: ", path, ", err: ", err) + return err + } else { + logs.Info("PatchWebHooks, Create webhook successfully, hookId: ", hookId, ", path: ", path) + } + } + return nil + } + return errors.New("创建仓库hook失败") +} + +func CreateDepositHooks(accessToken string, owner string, path string, + cve models.VulnCenter, issueNum string, issueId int64) error { + createFlag, ok := beego.AppConfig.Int("hook::create_hook") + if ok != nil { + createFlag = 1 + } + if createFlag == 1 { + var ih models.IssueHooks + ih.CveId = cve.CveId + ih.IssueNum = issueNum + ih.Owner = owner + ih.Status = 1 + ih.Repo = path + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("CreateDepositHooks, config init, error:", err) + return err + } + pwd := BConfig.String("hook::hookpwd") + hookurl := BConfig.String("hook::hookurl") + ihs, errh := models.GetIssueHook(&ih) + if errh { + hookExist := false + for _, wh := range ihs { + if wh.HookUrl == hookurl { + hookExist = true + hookId := strconv.FormatInt(wh.HookId, 10) + patchErr := PatchWebHooks(accessToken, owner, path, hookurl, + pwd, issueNum, hookId, issueId, cve.CveId) + if patchErr == nil { + logs.Info("CreateDepositHooks, owner: ", owner, + ",repo: ", path, ",webhook updated successfully") + } else { + logs.Error("CreateDepositHooks, owner: ", owner, + ",repo: ", path, ",patchErr: ", patchErr) + } + } + } + if !hookExist { + postErr := PostWebHooks(accessToken, owner, path, + hookurl, pwd, issueNum, issueId, cve.CveId) + if postErr == nil { + logs.Info("CreateDepositHooks, owner: ", owner, + ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("CreateDepositHooks, owner: ", owner, + ",repo: ", path, ",postErr: ", postErr) + } + } + } else { + postErr := PostWebHooks(accessToken, owner, path, + hookurl, pwd, issueNum, issueId, cve.CveId) + if postErr == nil { + logs.Info("CreateDepositHooks, owner: ", owner, + ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("CreateDepositHooks, owner: ", owner, + ",repo: ", path, ", postErr: ", postErr) + } + } + } + return nil +} + +func CreateIssueComment(accessToken, owner, path, assignee string, + cve models.VulnCenter, issueNum, affectedVersion string, assignLoginList []string) error { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/issues/" + issueNum + "/comments" + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("CreateIssueComment, config init, error:", err) + return err + } + commentCmd := BConfig.String("reflink::comment_cmd") + commentBody := "" + if cve.OrganizationID == 4 { + commentCmd = BConfig.String("reflink::looKeng_comment_cmd") + commentBody = LooKengCommentTemplate(assignee, commentCmd, affectedVersion) + } else if cve.OrganizationID == 3 { + commentCmd = BConfig.String("reflink::spore_comment_cmd") + commentBody = SporeCommentTemplate(assignee, commentCmd, affectedVersion) + } else if cve.OrganizationID == 2 { + commentCmd = BConfig.String("reflink::gauss_comment_cmd") + commentBody = GaussCommentTemplate(assignee, commentCmd, affectedVersion) + } else { + commentBody = CommentTemplate(assignee, commentCmd, affectedVersion, path, assignLoginList) + } + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "body": "%s" + }`, accessToken, commentBody) + logs.Info("CreateIssueComment, create issue comment body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("CreateIssueComment, Failed to create issue comment, url: ", url, + ", cveId", cve.CveId, ",issueNum: ", issueNum, ",err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("CreateIssueComment, Failed to create issue comment, err: ", ok, ", url: ", url) + return errors.New("创建issue评论失败") + } + commentID := int64(resp["id"].(float64)) + models.UpdateIssueCommentId(issueNum, cve.CveNum, commentID) + } + return nil +} + +func AddAffectBrands(branchVersion string) string { + branchs := "" + if branchVersion != "" && len(branchVersion) > 1 { + brandsGroup := strings.Split(branchVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "受影响" { + branchs += brandList[0] + "/" + } + } else { + brandList := strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "受影响" { + branchs += brandList[0] + "/" + } + } + } + } + } + } + if branchs != "" && len(branchs) > 1 { + branchs = branchs[:len(branchs)-1] + } + return branchs +} + +func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, + path, branchVersion string, opScore float64) { + branchs := AddAffectBrands(branchVersion) + sec.CveId = iss.CveId + sec.CveNum = iss.CveNum + opScoreLeve := models.OpenEulerScoreProc(opScore) + if len(iss.RepoName) < 1 { + iss.RepoName = iss.PackName + } + sec.Introduction = "An update for " + iss.RepoName + " is now available for " + branchs + "." + if iss.OrganizationID == 4 { + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openLooKeng Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + } else if iss.OrganizationID == 3 { + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "MindSpore Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + } else if iss.OrganizationID == 2 { + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openGauss Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + } else { + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + } + sec.AffectProduct = branchs +} + +func CreateBrandAndTags(accessToken, owner, path string, organizationID int8) []string { + tagsList := make([]string, 0) + branchList, errBrands := GetBranchesInfo(accessToken, owner, path, organizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("ProcIssue, Failed to obtain branch information,CveNum: ", "", + ", path: ", path, ", err: ", errBrands) + } + if len(branchList) > 0 { + tagList, tagErr := GetTagsInfo(accessToken, owner, path, organizationID) + if tagList == nil || len(tagList) == 0 { + logs.Error("GetTagsInfo, Failed to obtain tag information, path: ", path, ",owner:", owner, ", err: ", tagErr) + tagsList = append(tagsList, "master") + } else { + if organizationID == 3 { + for _, branch := range branchList { + // delete data + models.DelMindSporeBrandTagsByBrand(branch, path) + bl := common.StripStrReg(branch) + for _, tags := range tagList { + if strings.Contains(tags, bl) { + mbt := models.MindSporeBrandTags{PackageName: path, Brand: branch, Tags: tags, CreateTime: common.GetCurTime()} + spId, spErr := models.InsertMindSporeBrandTags(&mbt) + if spErr != nil { + logs.Error(spId, spErr) + continue + } + tagsList = append(tagsList, tags) + } + } + if branch == "master" { + mbt := models.MindSporeBrandTags{PackageName: path, Brand: branch, Tags: branch, CreateTime: common.GetCurTime()} + spId, spErr := models.InsertMindSporeBrandTags(&mbt) + if spErr != nil { + logs.Error(spId, spErr) + continue + } + tagsList = append(tagsList, branch) + } + } + } else if organizationID == 4 { + tagsList = append(tagsList, tagList...) + tagsList = append(tagsList, "master") + } + } + } + repList := make([]string, 0) + if len(tagsList) > 0 { + sort.Strings(tagsList) + tmpList := common.SliceRemoveDup(tagsList) + for _, tl := range tmpList { + repList = append(repList, tl.(string)) + } + tagsList = repList + } + logs.Info("CreateBrandAndTags, tagsList: ", tagsList) + return tagsList +} diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go new file mode 100644 index 0000000000000000000000000000000000000000..acc0a9ddb380e06a0166943dc35322a9ba403fa5 --- /dev/null +++ b/cve-vulner-manager/taskhandler/cve.go @@ -0,0 +1,2637 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +var GetCveDetailUrl = "https://openeuler.org/api-cve/cve-security-notice-server/cvedatabase/getByCveId?cveId=%s" + +var ( + pkgLock sync.Mutex + lockx sync.Mutex + dbLock sync.Mutex +) +var ewg sync.WaitGroup + +type UpdateInfoXml struct { + XmfilePath string + Cvexml []CveXml + Dpdates *Updates + SecurityNotice map[string][]SecurityNoticeXml + PackRpmx map[string][]PackRpm + FileName string + AffectBranch string +} + +type PackageVersion struct { + OrganizationID int8 + VerionList []string +} + +func UpdateExcelCveGroups(cveData models.OriginExcel, + cveRef string, openeulerNum int, CveRes models.VulnCenter, + goe models.GitPackageInfo, organizationID int8) (bool, error) { + var OpenEulId int64 + CveRes.Description = cveData.CveDesc + CveRes.CveVersion = cveData.CveVersion + if len(CveRes.RepairTime) < 2 && len(cveData.RepairTime) > 2 { + CveRes.RepairTime = cveData.RepairTime + } + if len(goe.Decription) < 1 { + goe.Decription = goe.Summary + } + //CveRes.PackName = cveData.PackName + CveRes.RepoName = cveData.PackName + CveRes.CveUrl = cveRef + cveData.CveNum + CveRes.CveLevel = cveData.CveLevel + CveRes.OrganizationID = organizationID + if CveRes.DataSource == 1 { + CveRes.DataSource = 3 + } + if CveRes.DataSource != 4 && CveRes.Status != 0 && CveRes.Status != 1 { + CveRes.Status = 1 + CveRes.IsExport = 0 + } + if (CveRes.DataSource == 4 || CveRes.DataSource == 3) && + CveRes.Status != 0 && CveRes.Status != 1 { + issueTmp := models.IssueTemplate{CveId: CveRes.CveId, + Repo: CveRes.PackName, OwnedVersion: CveRes.CveVersion} + err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "Repo", "OwnedVersion") + if err != nil { + if CveRes.DataSource == 3 { + CveRes.Status = 0 + CveRes.IsExport = 0 + } + logs.Error(err) + } else { + if issueTmp.Status < 3 { + CveRes.Status = 1 + CveRes.IsExport = 0 + } + } + } + //CveRes.Status = 1 + openEusa, operr := models.QueryOpenEulerSAByCveId(CveRes.CveId) + if operr == false { + var opensa models.OpenEulerSA + osx, operr := models.QueryOpenSaLastId() + var OpenNumData int + if operr == nil { + OpenNumList := strings.Split(osx.OpenEulerSANum, "-") + OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) + if err == nil { + OpenNum += 1 + } else { + OpenNum = openeulerNum + } + OpenNumData = OpenNum + } else { + OpenNumData = openeulerNum + } + OpenEulerSANum := "openEuler-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.Itoa(OpenNumData) + opensa.OpenEulerSANum = OpenEulerSANum + opensa.CveId = CveRes.CveId + openEulerId, opEerr := models.CreateOpenEulerSA(&opensa) + if opEerr == nil { + OpenEulId = openEulerId + } else { + logs.Error("UpdateExcelCveGroups, Failed to update openEulerSa data table, err: ", opEerr) + return false, errors.New("openEulerSA数据错误，暂时不处理") + } + } else { + OpenEulId = openEusa.OpenId + } + scoreRes, scoreerr := models.QueryScoreByCveId(CveRes.CveId) + if scoreerr { + if scoreRes.NVDScore != cveData.NVDScore { + var scorecode models.ScoreRecord + scorecode.NVDScore = cveData.NVDScore + scorecode.NvectorVule = cveData.NVDVector + scorecode.Status = 0 + scorecode.CveId = CveRes.CveId + scoreid, err := models.InsertScoreRecord(&scorecode) + if scoreid > 0 && err == nil { + logs.Info("insert score_record success, id:", scoreid) + } else { + return false, errors.New("评分记录数据错误，暂时不处理") + } + } + scoreRes.NVDScore = cveData.NVDScore + scoreRes.NvectorVule = cveData.NVDVector + scoreRes.OpenId = OpenEulId + scoreRes.Nstatus = 1 + if strings.ToLower(cveData.ScoreType) == "v2" || strings.ToLower(cveData.ScoreType) == "v2.0" { + scoreRes.NaccessVector = cveData.AccessVector + scoreRes.NaccessComplexity = cveData.AccessComplexity + scoreRes.Nauthentication = cveData.Authentication + scoreRes.ScoreType = "v2" + } else { + scoreRes.NattackVector = cveData.AttackVector + scoreRes.NattackComplexity = cveData.AttackComplexity + scoreRes.NprivilegeRequired = cveData.PrivilegeRequired + scoreRes.NuserInteraction = cveData.UserInteraction + scoreRes.Nscope = cveData.Scope + scoreRes.ScoreType = "v3" + } + scoreRes.Nconfidentiality = cveData.Confidentiality + scoreRes.Nintegrity = cveData.Integrity + scoreRes.Navailability = cveData.Availability + } else { + var sc models.Score + sc.CveNum = cveData.CveNum + sc.NVDScore = cveData.NVDScore + sc.OpenEulerScore = cveData.NVDScore + sc.OpenId = OpenEulId + sc.NvectorVule = cveData.NVDVector + sc.OvectorVule = cveData.NVDVector + sc.CveId = CveRes.CveId + sc.Nstatus = 0 + sc.Ostatus = 0 + if strings.ToLower(cveData.ScoreType) == "v2" || + strings.ToLower(cveData.ScoreType) == "v2.0" { + sc.NaccessVector = cveData.AccessVector + sc.OaccessVector = cveData.AccessVector + sc.NaccessComplexity = cveData.AccessComplexity + sc.OaccessComplexity = cveData.AccessComplexity + sc.Nauthentication = cveData.Authentication + sc.Oauthentication = cveData.Authentication + sc.ScoreType = "v2" + } else { + sc.NattackVector = cveData.AttackVector + sc.OattackVector = cveData.AttackVector + sc.NattackComplexity = cveData.AttackComplexity + sc.OattackComplexity = cveData.AttackComplexity + sc.NprivilegeRequired = cveData.PrivilegeRequired + sc.OprivilegeRequired = cveData.PrivilegeRequired + sc.OuserInteraction = cveData.UserInteraction + sc.NuserInteraction = cveData.UserInteraction + sc.Oscope = cveData.Scope + sc.Nscope = cveData.Scope + sc.ScoreType = "v3" + } + sc.Nconfidentiality = cveData.Confidentiality + sc.Oconfidentiality = cveData.Confidentiality + sc.Nintegrity = cveData.Integrity + sc.Ointegrity = cveData.Integrity + sc.Navailability = cveData.Availability + sc.Oavailability = cveData.Availability + scid, scErr := models.CreateScore(&sc) + if scErr != nil { + logs.Error("UpdateExcelCveGroups, insert cve_score failed cveScV3: ", sc, ",scerr: ", scErr) + return false, errors.New("记录评分失败，暂时不处理") + } else { + logs.Info("insert cve_score success scid: ", scid, "CveNum:", cveData.CveNum) + } + } + SecNOtice, secerrx := models.QuerySecNoticeByCveId(CveRes.CveId) + if secerrx { + SecNOtice.InfluenceComponent = cveData.PackName + SecNOtice.OpenId = OpenEulId + SecNOtice.Summary = cveData.PackName + " security update" + SecNOtice.ReferenceLink = cveRef + cveData.CveNum + } else { + var sec models.SecurityNotice + sec.CveNum = cveData.CveNum + sec.OpenId = OpenEulId + sec.InfluenceComponent = cveData.PackName + sec.Status = 0 + sec.AffectStatus = "Fixed" + sec.CveId = CveRes.CveId + sec.Summary = cveData.PackName + " security update" + sec.ReferenceLink = cveRef + cveData.CveNum + if goe.Decription != "" && len(goe.Decription) > 0 { + sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + cveData.CveDesc + "(" + CveRes.CveNum + ")" + } else { + sec.Description = "Security Fix(es):" + "\n\n" + cveData.CveDesc + "(" + CveRes.CveNum + ")" + } + secid, secerr := models.CreateSecurityNotice(&sec) + if secerr != nil { + logs.Error("insert cve_security_notice failed CveNum:", cveData.CveNum) + return false, errors.New("记录SA失败，暂时不处理") + } else { + logs.Info("insert cve_security_notice success secid: , cveNum: ", secid, cveData.CveNum) + } + } + if scoreerr && secerrx { + errx := models.UpdateCveRelat(&CveRes, &SecNOtice, &scoreRes) + if errx != nil { + logs.Error("UpdateExcelCveGroups, UpdateCveRelat, update failed, CveNum:", + cveData.CveNum, ", errx: ", errx) + return false, errors.New("数据更新失败, 暂时不处理") + } + } else if scoreerr { + errx := models.UpdateCveRelat1(&CveRes, &SecNOtice) + if errx != nil { + logs.Error("UpdateExcelCveGroups, UpdateCveRelat1, update failed, CveNum:", + cveData.CveNum, ", errx: ", errx) + return false, errors.New("数据更新失败, 暂时不处理") + } + } else { + errx := models.UpdateCveRelat2(&CveRes, &scoreRes) + if errx != nil { + logs.Error("UpdateExcelCveGroups, UpdateCveRelat2, "+ + "update failed, CveNum:", cveData.CveNum, ",errx: ", errx) + return false, errors.New("数据更新失败, 暂时不处理") + } + } + return true, nil +} + +func InsertCveExcelGroups(cveData models.OriginExcel, cveRef, repoName string, openeulerNum int, + goe models.GitPackageInfo, organizationID int8) (bool, error) { + var vul models.VulnCenter + vul.CveNum = cveData.CveNum + vul.Description = cveData.CveDesc + vul.Status = 0 + vul.CveVersion = cveData.CveVersion + vul.RepairTime = cveData.RepairTime + vul.PackName = repoName + vul.RepoName = cveData.PackName + vul.CveUrl = cveRef + cveData.CveNum + vul.CveLevel = cveData.CveLevel + vul.DataSource = 3 + vul.IsExport = 0 + vul.OrganizationID = organizationID + var sec models.SecurityNotice + sec.CveNum = cveData.CveNum + sec.InfluenceComponent = cveData.PackName + sec.Status = 0 + sec.AffectStatus = "Fixed" + sec.Summary = cveData.PackName + " security update" + sec.ReferenceLink = cveRef + cveData.CveNum + specCharList := []string{"\n", "\r", "\t"} + if len(goe.Decription) < 1 { + goe.Decription = goe.Summary + } + if goe.Decription != "" && len(goe.Decription) > 0 { + sec.Description = RemoveSubstring(goe.Decription, specCharList) + "\n\n" + + "Security Fix(es):" + "\n\n" + RemoveSubstring(cveData.CveDesc, specCharList) + } else { + sec.Description = "Security Fix(es):" + "\n\n" + RemoveSubstring(cveData.CveDesc, specCharList) + } + if sec.Description != "" && len(sec.Description) > 1 { + sec.Description = deleteTailBlank(sec.Description) + "(" + vul.CveNum + ")" + } else { + sec.Description += "(" + vul.CveNum + ")" + } + var sc models.Score + sc.CveNum = cveData.CveNum + sc.NVDScore = cveData.NVDScore + sc.OpenEulerScore = cveData.NVDScore + sc.NvectorVule = cveData.NVDVector + sc.OvectorVule = cveData.NVDVector + sc.Nstatus = 0 + sc.Ostatus = 0 + if strings.ToLower(cveData.ScoreType) == "v2" || + strings.ToLower(cveData.ScoreType) == "v2.0" { + sc.NaccessVector = cveData.AccessVector + sc.OaccessVector = cveData.AccessVector + sc.NaccessComplexity = cveData.AccessComplexity + sc.OaccessComplexity = cveData.AccessComplexity + sc.Nauthentication = cveData.Authentication + sc.Oauthentication = cveData.Authentication + sc.ScoreType = "v2" + } else { + sc.NattackVector = cveData.AttackVector + sc.OattackVector = cveData.AttackVector + sc.NattackComplexity = cveData.AttackComplexity + sc.OattackComplexity = cveData.AttackComplexity + sc.NprivilegeRequired = cveData.PrivilegeRequired + sc.OprivilegeRequired = cveData.PrivilegeRequired + sc.OuserInteraction = cveData.UserInteraction + sc.NuserInteraction = cveData.UserInteraction + sc.Oscope = cveData.Scope + sc.Nscope = cveData.Scope + sc.ScoreType = "v3" + } + sc.Nconfidentiality = cveData.Confidentiality + sc.Oconfidentiality = cveData.Confidentiality + sc.Nintegrity = cveData.Integrity + sc.Ointegrity = cveData.Integrity + sc.Navailability = cveData.Availability + sc.Oavailability = cveData.Availability + var opensa models.OpenEulerSA + //var os models.OpenSaId + osx, operr := models.QueryOpenSaLastId() + var OpenNumData int + if operr == nil { + OpenNumList := strings.Split(osx.OpenEulerSANum, "-") + OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) + if err == nil { + OpenNum += 1 + } else { + OpenNum = openeulerNum + } + OpenNumData = OpenNum + } else { + OpenNumData = openeulerNum + } + OpenEulerSANum := "openEuler-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.Itoa(int(OpenNumData)) + opensa.OpenEulerSANum = OpenEulerSANum + var scorecode models.ScoreRecord + scorecode.NVDScore = cveData.NVDScore + scorecode.NvectorVule = cveData.NVDVector + scorecode.Status = 0 + cveid, cveerr := models.CreateCveRelat(&vul, &sec, &sc, &opensa, &scorecode) + if cveerr != nil || cveid <= 0 { + logs.Error("InsertCveExcelGroups, CreateCveRelat, "+ + "insert failed, CveNum:", cveData.CveNum, ", cveerr: ", cveerr) + return false, errors.New("数据插入失败, 暂时不处理") + } + return true, nil +} + +func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum int, CveRes models.VulnCenter, + cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, + scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, pkList []string, organizationID int8) (bool, error) { + var OpenEulId int64 + if len(cveDesc.EnDescription) > 2 { + CveRes.Description = cveDesc.EnDescription + } + CveRes.CveVersion = pkList[1] + if len(CveRes.RepairTime) < 2 && len(cveData.PublishedDate) > 2 { + CveRes.RepairTime = cveData.PublishedDate + } + //CveRes.PackName = pkList[0] + CveRes.RepoName = pkList[0] + CveRes.CveUrl = cveRef + cveData.CveNum + CveRes.CveLevel = cveScV3.CveLevel + CveRes.OrganizationID = organizationID + CveRes.FirstPerTime = cveData.FirstPerTime + CveRes.FirstGetTime = cveData.FirstGetTime + //CveRes.DataSource = 1 + if CveRes.DataSource != 4 && CveRes.Status != 0 && CveRes.Status != 1 { + CveRes.Status = 1 + CveRes.IsExport = 0 + } + //CveRes.Status = 1 + openEusa, operr := models.QueryOpenEulerSAByCveId(CveRes.CveId) + if operr == false { + var opensa models.OpenEulerSA + osx, operr := models.QueryOpenSaLastId() + var OpenNumData int + if operr == nil { + OpenNumList := strings.Split(osx.OpenEulerSANum, "-") + OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) + if err == nil { + OpenNum += 1 + } else { + OpenNum = openeulerNum + } + OpenNumData = OpenNum + } else { + OpenNumData = openeulerNum + } + OpenEulerSANum := "openEuler-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.Itoa(OpenNumData) + opensa.OpenEulerSANum = OpenEulerSANum + opensa.CveId = CveRes.CveId + openEulerId, opEerr := models.CreateOpenEulerSA(&opensa) + if opEerr == nil { + OpenEulId = openEulerId + } else { + logs.Error("InsertCveExcelGroups, CreateOpenEulerSA, Update failed, err: ", opEerr) + return false, errors.New("openEulerSA数据错误，暂时不处理") + } + } else { + OpenEulId = openEusa.OpenId + } + scoreRes, scoreerr := models.QueryScoreByCveId(CveRes.CveId) + if scoreerr { + scoreRes.ScoreType = scopeType + vectorString := "" + if scopeType == "v3" { + if cveScV3.VectorString != "" && len(cveScV3.VectorString) > 0 { + index := strings.IndexAny(cveScV3.VectorString, "/") + if index > 0 && strings.ToLower(cveScV3.VectorString)[:4] == "cvss" { + vectorString = cveScV3.VectorString[index+1:] + } else { + vectorString = cveScV3.VectorString + } + } + if scoreRes.NVDScore != cveScV3.BaseScore { + var scorecode models.ScoreRecord + scorecode.NVDScore = cveScV3.BaseScore + scorecode.NvectorVule = vectorString + scorecode.Status = 0 + scorecode.CveId = CveRes.CveId + scoreid, err := models.InsertScoreRecord(&scorecode) + if scoreid > 0 && err == nil { + logs.Info("insert score_record success, id:", scoreid) + } else { + return false, errors.New("评分记录数据错误，暂时不处理") + } + } + scoreRes.NVDScore = cveScV3.BaseScore + scoreRes.NvectorVule = vectorString + scoreRes.OpenId = OpenEulId + scoreRes.Nstatus = 1 + scoreRes.NattackVector = cveScV3.AttackVector + scoreRes.NattackComplexity = cveScV3.AttackComplexity + scoreRes.NprivilegeRequired = cveScV3.PrivilegesRequired + scoreRes.NuserInteraction = cveScV3.UserInteraction + scoreRes.Nscope = cveScV3.Scope + scoreRes.Nconfidentiality = cveScV3.ConfidentialityImpact + scoreRes.Nintegrity = cveScV3.IntegrityImpact + scoreRes.Navailability = cveScV3.AvailabilityImpact + } else { + if cveScV2.VectorString != "" && len(cveScV2.VectorString) > 0 { + index := strings.IndexAny(cveScV2.VectorString, "/") + if index > 0 && strings.ToLower(cveScV2.VectorString)[:4] == "cvss" { + vectorString = cveScV2.VectorString[index+1:] + } else { + vectorString = cveScV2.VectorString + } + } + if scoreRes.NVDScore != cveScV2.BaseScore { + var scorecode models.ScoreRecord + scorecode.NVDScore = cveScV2.BaseScore + scorecode.NvectorVule = vectorString + scorecode.Status = 0 + scorecode.CveId = CveRes.CveId + scoreid, err := models.InsertScoreRecord(&scorecode) + if scoreid > 0 && err == nil { + logs.Info("insert score_record success, id:", scoreid) + } else { + return false, errors.New("评分记录数据错误，暂时不处理") + } + } + scoreRes.NVDScore = cveScV2.BaseScore + scoreRes.NvectorVule = vectorString + scoreRes.OpenId = OpenEulId + scoreRes.Nstatus = 1 + scoreRes.NaccessVector = cveScV2.AccessVector + scoreRes.NaccessComplexity = cveScV2.AccessComplexity + scoreRes.Nconfidentiality = cveScV2.ConfidentialityImpact + scoreRes.Nintegrity = cveScV2.IntegrityImpact + scoreRes.Navailability = cveScV2.AvailabilityImpact + scoreRes.Nauthentication = cveScV2.Authentication + } + } else { + var sc models.Score + sc.CveNum = cveData.CveNum + sc.OpenId = OpenEulId + sc.CveId = CveRes.CveId + sc.Nstatus = 0 + sc.Ostatus = 0 + sc.ScoreType = scopeType + if scopeType == "v3" { + sc.NVDScore = cveScV3.BaseScore + sc.OpenEulerScore = cveScV3.BaseScore + vectorString := "" + if cveScV3.VectorString != "" && len(cveScV3.VectorString) > 0 { + index := strings.IndexAny(cveScV3.VectorString, "/") + if index > 0 && strings.ToLower(cveScV3.VectorString)[:4] == "cvss" { + vectorString = cveScV3.VectorString[index+1:] + } else { + vectorString = cveScV3.VectorString + } + } + sc.NvectorVule = vectorString + sc.OvectorVule = vectorString + sc.NattackVector = cveScV3.AttackVector + sc.OattackVector = cveScV3.AttackVector + sc.NattackComplexity = cveScV3.AttackComplexity + sc.OattackComplexity = cveScV3.AttackComplexity + sc.NprivilegeRequired = cveScV3.PrivilegesRequired + sc.OprivilegeRequired = cveScV3.PrivilegesRequired + sc.NuserInteraction = cveScV3.UserInteraction + sc.OuserInteraction = cveScV3.UserInteraction + sc.Nscope = cveScV3.Scope + sc.Oscope = cveScV3.Scope + sc.Nconfidentiality = cveScV3.ConfidentialityImpact + sc.Oconfidentiality = cveScV3.ConfidentialityImpact + sc.Nintegrity = cveScV3.IntegrityImpact + sc.Ointegrity = cveScV3.IntegrityImpact + sc.Navailability = cveScV3.AvailabilityImpact + sc.Oavailability = cveScV3.AvailabilityImpact + } else { + sc.NVDScore = cveScV2.BaseScore + sc.OpenEulerScore = cveScV2.BaseScore + vectorString := "" + if cveScV2.VectorString != "" && len(cveScV2.VectorString) > 0 { + index := strings.IndexAny(cveScV2.VectorString, "/") + if index > 0 && strings.ToLower(cveScV2.VectorString)[:4] == "cvss" { + vectorString = cveScV2.VectorString[index+1:] + } else { + vectorString = cveScV2.VectorString + } + } + sc.NvectorVule = vectorString + sc.OvectorVule = vectorString + sc.NaccessVector = cveScV2.AccessVector + sc.OaccessVector = cveScV2.AccessVector + sc.NaccessComplexity = cveScV2.AccessComplexity + sc.OaccessComplexity = cveScV2.AccessComplexity + sc.Nauthentication = cveScV2.Authentication + sc.Oauthentication = cveScV2.Authentication + sc.Nconfidentiality = cveScV2.ConfidentialityImpact + sc.Oconfidentiality = cveScV2.ConfidentialityImpact + sc.Nintegrity = cveScV2.IntegrityImpact + sc.Ointegrity = cveScV2.IntegrityImpact + sc.Navailability = cveScV2.AvailabilityImpact + sc.Oavailability = cveScV2.AvailabilityImpact + } + scid, scerr := models.CreateScore(&sc) + if scerr != nil { + logs.Error("InsertCveExcelGroups, CreateScore, "+ + "insert failed, cveScV3:", cveScV3, ",scerr: ", scerr) + return false, errors.New("记录评分失败，暂时不处理") + } else { + logs.Info("insert cve_score success scid: ", scid, "CveNum:", cveData.CveNum) + } + } + SecNOtice, secerrx := models.QuerySecNoticeByCveId(CveRes.CveId) + if secerrx { + SecNOtice.CveId = CveRes.CveId + SecNOtice.InfluenceComponent = pkList[0] + SecNOtice.OpenId = OpenEulId + SecNOtice.Summary = pkList[0] + " security update" + SecNOtice.ReferenceLink = cveRef + cveData.CveNum + } else { + var sec models.SecurityNotice + sec.CveNum = cveData.CveNum + sec.OpenId = OpenEulId + sec.InfluenceComponent = pkList[0] + sec.Status = 0 + sec.AffectStatus = "Fixed" + sec.CveId = CveRes.CveId + sec.Summary = pkList[0] + " security update" + sec.ReferenceLink = cveRef + cveData.CveNum + if len(goe.Decription) < 1 { + goe.Decription = goe.Summary + } + if goe.Decription != "" && len(goe.Decription) > 0 { + sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + cveDesc.EnDescription + "(" + CveRes.CveNum + ")" + } else { + sec.Description = "Security Fix(es):" + "\n\n" + cveDesc.EnDescription + "(" + CveRes.CveNum + ")" + } + secid, secerr := models.CreateSecurityNotice(&sec) + if secerr != nil { + logs.Error("InsertCveExcelGroups, CreateSecurityNotice, "+ + "insert failed, CveNum:", cveData.CveNum, ",secerr: ", secerr) + return false, errors.New("记录SA失败，暂时不处理") + } else { + logs.Info("insert cve_security_notice success secid: , cveNum: ", secid, cveData.CveNum) + } + } + if scoreerr && secerrx { + errx := models.UpdateCveRelat(&CveRes, &SecNOtice, &scoreRes) + if errx != nil { + logs.Error("InsertCveExcelGroups, UpdateCveRelat, "+ + "update failed, CveNum:", cveData.CveNum, ",errx: ", errx) + return false, errors.New("数据更新失败, 暂时不处理") + } + } else if scoreerr { + errx := models.UpdateCveRelat1(&CveRes, &SecNOtice) + if errx != nil { + logs.Error("InsertCveExcelGroups, UpdateCveRelat1, "+ + "update failed, CveNum:", cveData.CveNum, ",errx: ", errx) + return false, errors.New("数据更新失败, 暂时不处理") + } + } else { + errx := models.UpdateCveRelat2(&CveRes, &scoreRes) + if errx != nil { + logs.Error("InsertCveExcelGroups, UpdateCveRelat2, "+ + "update failed, CveNum:", cveData.CveNum, ", errx: ", errx) + return false, errors.New("数据更新失败, 暂时不处理") + } + } + return true, nil +} + +func InsertCveGroups(cveData models.OriginUpstream, cveRef, repoNme string, + openeulerNum int, cveDesc models.OriginUpstreamDesc, + cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, + scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, + pkList []string, organizationID int8) (bool, error) { + var vul models.VulnCenter + vul.CveNum = cveData.CveNum + vul.FirstPerTime = cveData.FirstPerTime + vul.FirstGetTime = cveData.FirstGetTime + vul.Description = cveDesc.EnDescription + vul.Status = 0 + vul.CveVersion = pkList[1] + vul.RepairTime = cveData.PublishedDate + vul.RepoName = pkList[0] + vul.PackName = repoNme + vul.CveUrl = cveRef + cveData.CveNum + vul.CveLevel = cveScV3.CveLevel + vul.DataSource = 1 + vul.IsExport = 0 + vul.OrganizationID = organizationID + var sec models.SecurityNotice + sec.CveNum = cveData.CveNum + sec.InfluenceComponent = pkList[0] + sec.Status = 0 + sec.AffectStatus = "Fixed" + sec.Summary = pkList[0] + " security update" + sec.ReferenceLink = cveRef + cveData.CveNum + specCharList := []string{"\n", "\r", "\t"} + if len(goe.Decription) < 1 { + goe.Decription = goe.Summary + } + if goe.Decription != "" && len(goe.Decription) > 0 { + sec.Description = RemoveSubstring(goe.Decription, specCharList) + "\n\n" + "Security Fix(es):" + "\n\n" + + RemoveSubstring(cveDesc.EnDescription, specCharList) + } else { + sec.Description = "Security Fix(es):" + "\n\n" + RemoveSubstring(cveDesc.EnDescription, specCharList) + } + if sec.Description != "" && len(sec.Description) > 1 { + sec.Description = deleteTailBlank(sec.Description) + "(" + vul.CveNum + ")" + } else { + sec.Description += "(" + vul.CveNum + ")" + } + var sc models.Score + sc.CveNum = cveData.CveNum + sc.Nstatus = 0 + sc.Ostatus = 0 + if scopeType == "v3" { + sc.NVDScore = cveScV3.BaseScore + sc.OpenEulerScore = cveScV3.BaseScore + vectorString := "" + if cveScV3.VectorString != "" && len(cveScV3.VectorString) > 0 { + index := strings.IndexAny(cveScV3.VectorString, "/") + if index > 0 && strings.ToLower(cveScV3.VectorString)[:4] == "cvss" { + vectorString = cveScV3.VectorString[index+1:] + } else { + vectorString = cveScV3.VectorString + } + } + sc.NvectorVule = vectorString + sc.OvectorVule = vectorString + sc.NattackVector = cveScV3.AttackVector + sc.OattackVector = cveScV3.AttackVector + sc.NattackComplexity = cveScV3.AttackComplexity + sc.OattackComplexity = cveScV3.AttackComplexity + sc.NprivilegeRequired = cveScV3.PrivilegesRequired + sc.OprivilegeRequired = cveScV3.PrivilegesRequired + sc.NuserInteraction = cveScV3.UserInteraction + sc.OuserInteraction = cveScV3.UserInteraction + sc.Nscope = cveScV3.Scope + sc.Oscope = cveScV3.Scope + sc.Nconfidentiality = cveScV3.ConfidentialityImpact + sc.Oconfidentiality = cveScV3.ConfidentialityImpact + sc.Nintegrity = cveScV3.IntegrityImpact + sc.Ointegrity = cveScV3.IntegrityImpact + sc.Navailability = cveScV3.AvailabilityImpact + sc.Oavailability = cveScV3.AvailabilityImpact + } else { + sc.NVDScore = cveScV2.BaseScore + sc.OpenEulerScore = cveScV2.BaseScore + vectorString := "" + if cveScV2.VectorString != "" && len(cveScV2.VectorString) > 0 { + index := strings.IndexAny(cveScV2.VectorString, "/") + if index > 0 && strings.ToLower(cveScV2.VectorString)[:4] == "cvss" { + vectorString = cveScV2.VectorString[index+1:] + } else { + vectorString = cveScV2.VectorString + } + } + sc.NvectorVule = vectorString + sc.OvectorVule = vectorString + sc.NaccessVector = cveScV2.AccessVector + sc.OaccessVector = cveScV2.AccessVector + sc.NaccessComplexity = cveScV2.AccessComplexity + sc.OaccessComplexity = cveScV2.AccessComplexity + sc.Nauthentication = cveScV2.Authentication + sc.Oauthentication = cveScV2.Authentication + sc.Nconfidentiality = cveScV2.ConfidentialityImpact + sc.Oconfidentiality = cveScV2.ConfidentialityImpact + sc.Nintegrity = cveScV2.IntegrityImpact + sc.Ointegrity = cveScV2.IntegrityImpact + sc.Navailability = cveScV2.AvailabilityImpact + sc.Oavailability = cveScV2.AvailabilityImpact + } + sc.ScoreType = scopeType + var opensa models.OpenEulerSA + //var os models.OpenSaId + osx, operr := models.QueryOpenSaLastId() + var OpenNumData int + if operr == nil { + OpenNumList := strings.Split(osx.OpenEulerSANum, "-") + OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) + if err == nil { + OpenNum += 1 + } else { + OpenNum = openeulerNum + } + OpenNumData = OpenNum + } else { + OpenNumData = openeulerNum + } + OpenEulerSANum := "openEuler-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.Itoa(int(OpenNumData)) + opensa.OpenEulerSANum = OpenEulerSANum + var scorecode models.ScoreRecord + scorecode.NVDScore = cveScV3.BaseScore + scorecode.NvectorVule = cveScV3.VectorString + scorecode.Status = 0 + cveId, cveErr := models.CreateCveRelat(&vul, &sec, &sc, &opensa, &scorecode) + if cveErr != nil || cveId <= 0 { + logs.Error("InsertCveGroups, CreateCveRelat, insert failed, "+ + "CveNum:", cveData.CveNum, ",cveErr: ", cveErr) + return false, errors.New("数据插入失败, 暂时不处理") + } + return true, nil +} + +// Synchronize the data returned by the Chinese Academy of Sciences +func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) (bool, error) { + if cveData.Ids == "" || cveData.CveNum == "" { + logs.Error("GenCveVuler, The current data cveNum is empty "+ + "and will not be processed temporarily, data: ", cveData) + models.UpdateOriginStatus(common.GetCurTime(), + cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "ids or cveNum is empty", 3) + return false, errors.New("数据错误，暂时不处理") + } + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("GenCveVuler, config init error:", err, cveData.CveNum) + models.UpdateOriginStatus(common.GetCurTime(), + cveData.PackName, cveData.Version, cveData.CveId, 0) + return false, err + } + years, confOk := BConfig.Int("cve::cve_number_t") + if confOk != nil { + logs.Error("GenCveVuler, config cve::cve_number_t, error:", confOk, cveData.CveNum) + models.UpdateOriginStatus(common.GetCurTime(), + cveData.PackName, cveData.Version, cveData.CveId, 0) + return false, errors.New("数据错误，暂时不处理") + } + // Import cve as data after 2018 + cveNumList := strings.Split(cveData.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if cveYears <= years { + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 5) + logs.Error("GenCveVuler, cve: ", cveData.CveNum, ",Need to be greater than: ", years, + ", Otherwise there is no need to submit an issue on git, cveData: ", cveData) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, cveData.CveNum+" cve year limited", 5) + return false, errors.New("数据错误，暂时不处理") + } + } + } + cveDesc, ok := models.QueryCveDesc(cveData.CveId) + if !ok { + logs.Error("GenCveVuler, QueryCveDesc, The current data description is empty "+ + "and will not be processed temporarily, data: ", cveData) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve description information is empty", 3) + return false, errors.New("数据错误，暂时不处理") + } + cveImpact, ok := models.QueryCveImpact(cveData.CveId) + if !ok { + logs.Error("GenCveVuler, QueryCveImpact, Data query failed, data: ", cveData) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 1", 3) + return false, errors.New("数据错误，暂时不处理") + } + scopeType := "v3" + //var cveScV2 models.OriginUpstreamImpactScoreV2 + cveScore, ok := models.QueryCveScore(cveImpact.ImpactId, "v3") + if !ok { + scopeType = "v2" + } + cveScV3, ok := models.QueryCveCvssV3(cveScore.ScoreId) + if !ok { + scopeType = "v2" + } + if cveScV3.VectorString == "" || len(cveScV3.VectorString) == 0 || cveScV3.BaseScore == 0.0 { + scopeType = "v2" + } + cveScoreV2, ok2 := models.QueryCveScore(cveImpact.ImpactId, "v2") + if !ok2 { + if scopeType == "v2" { + logs.Error("GenCveVuler, QueryCveScore3, data: ", cveData, ",cveImpact: ", cveImpact) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 2", 3) + return false, errors.New("数据错误，暂时不处理") + } + } + cveScV2, okV2 := models.QueryCveCvssV2(cveScoreV2.ScoreId) + if !okV2 { + if scopeType == "v2" { + logs.Error("GenCveVuler, QueryCveCvssV2, data: ", cveData, ",cveScore: ", cveScore) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 3", 3) + return false, errors.New("数据错误，暂时不处理") + } + } + packNameMap := map[string]string{} + packNameList := []string{} + if cveData.PackName != "" && len(cveData.PackName) > 0 { + packNameList = strings.Split(cveData.PackName, ",") + if len(packNameList) > 0 { + for _, pk := range packNameList { + pkList := strings.Split(pk, "==") + if len(pkList) == 2 { + if _, ok := packNameMap[pkList[0]]; ok { + versionList := strings.Split(packNameMap[pkList[0]], ",") + verExist := false + for _, vl := range versionList { + if vl == pkList[1] { + verExist = true + break + } + } + if !verExist { + packNameMap[pkList[0]] += "," + pkList[1] + } + } else { + packNameMap[pkList[0]] = pkList[1] + } + } + } + } + } + for key, value := range packNameMap { + if key != "" && len(key) > 1 { + pkList := []string{} + pkList = append(pkList, key) + eulerFlag := false + gaussFlag := false + sporeFlag := false + looKengFlag := false + var goe models.GitPackageInfo + pvList := make([]PackageVersion, 0) + openGausVersion := make([]string, 0) + mindSporeVersion := make([]string, 0) + openEulerVersion := make([]string, 0) + openLooKengVersion := make([]string, 0) + versionList := make([]string, 0) + if value != "" && len(value) > 0 { + versionList = strings.Split(value, ",") + if len(versionList) > 0 { + for _, ver := range versionList { + if ver != "" && len(ver) > 0 { + opy := models.OpenGussYaml{PackageName: key, Version: ver} + gaussErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if gaussErr == nil && opy.Id > 0 { + openGausVersion = append(openGausVersion, ver) + gaussFlag = true + } + ms := models.MindSporeYaml{PackageName: key, Version: ver} + sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") + if sporeErr == nil && ms.Id > 0 { + mindSporeVersion = append(mindSporeVersion, ver) + sporeFlag = true + } + olky := models.OpenLookengYaml{PackageName: key, Version: ver} + looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") + if looKengErr == nil && olky.Id > 0 { + openLooKengVersion = append(openLooKengVersion, ver) + looKengFlag = true + } + goe, ok = models.QueryCveOpeneulerDetaildataByName(key, ver) + if ok { + eulerFlag = true + openEulerVersion = append(openEulerVersion, ver) + } else { + logs.Error("GenCveVuler, The corresponding warehouse data is not queried, data: ", cveData) + } + } + } + } + } + if !eulerFlag && !gaussFlag && !sporeFlag && !looKengFlag { + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 4) + logs.Error("GenCveVuler, The version information corresponds to the error, ", + cveData.PackName, cveData.Version, cveData.CveNum) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, + "The cve component or version information cannot be matched to the corresponding data", 4) + continue + } + organizationID := int8(1) + if eulerFlag { + pv := PackageVersion{OrganizationID: organizationID, VerionList: openEulerVersion} + pvList = append(pvList, pv) + } + if gaussFlag { + organizationID = int8(2) + pv := PackageVersion{OrganizationID: organizationID, VerionList: openGausVersion} + pvList = append(pvList, pv) + } + if sporeFlag { + organizationID = int8(3) + pv := PackageVersion{OrganizationID: organizationID, VerionList: mindSporeVersion} + pvList = append(pvList, pv) + } + if looKengFlag { + organizationID = int8(4) + pv := PackageVersion{OrganizationID: organizationID, VerionList: openLooKengVersion} + pvList = append(pvList, pv) + } + failFlag := false + ok, addErr := AddOrSelectToCenter(key, cveRef, scopeType, value, cveData, + pvList, pkList, openeulernum, cveDesc, cveScV3, goe, cveScV2) + if !ok || addErr != nil { + logs.Error("AddOrSelectToCenter, addErr: ", addErr) + failFlag = true + } + if failFlag { + logs.Error("Part of the data failed") + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, + "Version does not match, data synchronization failed", 3) + continue + } + } + } + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 2) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "success", 2) + return true, nil +} + +func AddOrSelectToCenter(packageName, cveRef, scopeType, value string, cveData models.OriginUpstream, + pvList []PackageVersion, pkList []string, openeulerNum int, + cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, + cveScV2 models.OriginUpstreamImpactScoreV2) (bool, error) { + for _, pv := range pvList { + organizationID := pv.OrganizationID + if organizationID == 4 { + looKengVersion := pv.VerionList + if len(looKengVersion) > 0 { + looKengMap := make(map[string]string) + for _, ver := range looKengVersion { + olky := models.OpenLookengYaml{PackageName: packageName, Version: ver} + msy, mErr := models.GetOpenLookengYamlAll(&olky) + if len(msy) > 0 { + for _, my := range msy { + if _, ok := looKengMap[my.Repo]; !ok { + looKengMap[my.Repo] = packageName + } + } + } else { + logs.Error("openLooKeng, mErr: ", mErr) + } + } + if len(looKengMap) > 0 { + for repo, _ := range looKengMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + looKengVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, pv.OrganizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + } + } + } + } + } else if organizationID == 3 { + mindSporeVersion := pv.VerionList + if len(mindSporeVersion) > 0 { + mindMap := make(map[string]string) + for _, ver := range mindSporeVersion { + ms := models.MindSporeYaml{PackageName: packageName, Version: ver} + msy, mErr := models.GetMindSporeYamlAll(&ms) + if len(msy) > 0 { + for _, my := range msy { + if _, ok := mindMap[my.Repo]; !ok { + mindMap[my.Repo] = packageName + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + } + } + if len(mindMap) > 0 { + for repo, _ := range mindMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + } + } + } + } + } else if organizationID == 2 { + openGausVersion := pv.VerionList + if len(openGausVersion) > 0 { + gaussMap := make(map[string]string) + for _, ver := range openGausVersion { + opy := models.OpenGussYaml{PackageName: packageName, Version: ver} + ogy, oErr := models.GetOpengaussYamlAll(&opy) + if len(ogy) > 0 { + for _, gy := range ogy { + if _, ok := gaussMap[gy.Repo]; !ok { + gaussMap[gy.Repo] = packageName + } + } + } else { + logs.Error("openGauss, dErr: ", oErr) + } + } + if len(gaussMap) > 0 { + for repo, _ := range gaussMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + openGausVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openGauss, dErr: ", dErr) + } + } + } + } + } else { + openEulerVersion := pv.VerionList + ok, dErr := AddOrDataToCenter(packageName, packageName, cveRef, scopeType, value, cveData, + openEulerVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openEuler, dErr: ", dErr) + return false, dErr + } + } + } + return true, nil +} + +func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cveData models.OriginUpstream, + versionList, pkList []string, openeulerNum int, + cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, + cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { + CveRes, err := models.QueryCveByNum(cveData.CveNum, repoNme, organizationID) + if err { + CveRes.RepoName = packageName + if CveRes.DataSource == 1 { + retVersion := " " + if len(versionList) > 0 { + retVersion = strings.Join(versionList, ",") + } + pkList = append(pkList, retVersion) + } else { + retVersion := AddCveVersion(organizationID, versionList, CveRes.CveVersion) + pkList = append(pkList, retVersion) + } + lockx.Lock() + ok, err := UpdateCveGroups(cveData, cveRef, openeulerNum, CveRes, cveDesc, cveScV3, goe, + scopeType, cveScV2, pkList, organizationID) + lockx.Unlock() + if !ok { + logs.Error("GenCveVuler, UpdateCveGroups, cveData: ", cveData, ", err: ", err) + //models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } else { + retVersion := AddCveVersion(organizationID, versionList, "") + if len(retVersion) > 0 { + pkList = append(pkList, retVersion) + } else { + pkList = append(pkList, " ") + } + lockx.Lock() + ok, err := InsertCveGroups(cveData, cveRef, repoNme, openeulerNum, cveDesc, cveScV3, goe, + scopeType, cveScV2, pkList, organizationID) + lockx.Unlock() + if !ok { + logs.Error("GenCveVuler, InsertCveGroups, cveData: ", cveData, ", err: ", err) + //models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } + return true, nil +} + +func UpdateOriginUpstreamRecord(ids, cveNum, errDesc string, errStatus int8) { + our := models.OriginUpstreamRecord{} + err := models.QueryOriginUpstreamRecord(&our, ids, cveNum) + if our.CveId > 0 { + our.SysStatus = errStatus + our.ErrDesc = errDesc + upErr := models.UpdateOriginUpstreamRecord(&our, "SysStatus", "ErrDesc") + if upErr != nil { + logs.Error("upErr: ", upErr) + } + } else { + logs.Error("err: ", err) + } +} + +func AddCveVersion(organizationID int8, versionList []string, cveVersion string) (retVersion string) { + retVersion = cveVersion + cveVerList := []string{} + if len(cveVersion) > 0 { + cveVerList = strings.Split(cveVersion, ",") + } + if organizationID != 1 { + if len(cveVerList) > 0 && len(versionList) > 0 { + versionList = append(versionList, cveVerList...) + } + verSlice := make([]string, 0) + if len(versionList) > 0 { + sort.Strings(versionList) + repSlice := common.SliceRemoveDup(versionList) + for _, rep := range repSlice { + if len(rep.(string)) > 0 { + verSlice = append(verSlice, rep.(string)) + } + } + } + if len(verSlice) > 0 { + retVersion = strings.Join(verSlice, ",") + } + } else { + if len(cveVerList) > 0 && len(versionList) > 0 { + versionList = append(versionList, cveVerList...) + } + eulerSlice := make([]string, 0) + if len(versionList) > 0 { + tmpEulerSlice := make([]string, len(versionList)) + for _, es := range versionList { + if len(es) > 0 { + versx := common.BranchVersionRep(es) + tmpEulerSlice = append(tmpEulerSlice, versx) + } + } + sort.Strings(tmpEulerSlice) + repSlice := common.SliceRemoveDup(tmpEulerSlice) + for _, rep := range repSlice { + if len(rep.(string)) > 0 { + eulerSlice = append(eulerSlice, rep.(string)) + } + } + } + if len(eulerSlice) > 0 { + retVersion = strings.Join(eulerSlice, ",") + } + } + if len(retVersion) > 0 { + verList := make([]string, 0) + tmpVer := strings.Split(retVersion, ",") + for _, ver := range tmpVer { + if len(ver) > 0 && ver != " " { + verList = append(verList, ver) + } + } + if len(verList) > 0 { + retVersion = strings.Join(verList, ",") + } else { + retVersion = " " + } + } + if len(retVersion) > 0 && retVersion[0] == ',' { + retVersion = retVersion[1:] + } + return retVersion +} + +func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYears int) (bool, error) { + if cveData.CveNum == "" || len(cveData.CveNum) == 0 { + logs.Error("SyncCveVuler, The current data cveNum is empty and "+ + "will not be processed temporarily, data: ", cveData) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + organizationID := int8(1) + organizationList := make([]int8, 0) + olky := models.OpenLookengYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") + if looKengErr == nil && olky.Id > 0 { + organizationID = 4 + organizationList = append(organizationList, organizationID) + } + ms := models.MindSporeYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") + if sporeErr == nil && ms.Id > 0 { + organizationID = 3 + organizationList = append(organizationList, organizationID) + } + opy := models.OpenGussYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + organizationID = 2 + organizationList = append(organizationList, organizationID) + } + var god models.GitOpenEuler + god.PackageName = cveData.PackName + god.Version = cveData.CveVersion + god.Status = 1 + ok := models.QueryCveOpeneulerdata(&god) + if !ok { + if organizationID == 1 { + logs.Error("The repo and version correspond to errors, data: ", cveData) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 4) + return false, errors.New("The repo and version correspond to errors and" + + " will not be processed temporarily") + } + } else { + organizationList = append(organizationList, organizationID) + } + // Import cve as data after 2018 + cveNumList := strings.Split(cveData.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if cveYears <= manYears { + logs.Error("limited cve years, cveData: ", cveData, ", err: ", cveYears, manYears) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 7) + return false, errors.New("limited cve years") + } + } + } + goe, ok := models.QueryCveOpeneulerDetaildataByName(cveData.PackName, cveData.CveVersion) + if !ok { + logs.Error("SyncCveVuler, The repo exists, but the yaml data source does not exist,"+ + " repo: ", cveData.PackName, cveData.CveVersion) + } + failFlag := false + for _, orId := range organizationList { + exOk, exErr := AddExcelToCenter(cveData, cveRef, openeulerNum, goe, orId) + if !exOk || exErr != nil { + logs.Error("AddExcelToCenter, exErr: ", exErr) + failFlag = true + } + } + if failFlag { + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 2) + return true, nil +} + +func AddExcelToCenter(cveData models.OriginExcel, cveRef string, openeulerNum int, + goe models.GitPackageInfo, organizationID int8) (bool, error) { + if organizationID == 4 { + olky := models.OpenLookengYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + opy, mErr := models.GetOpenLookengYamlAll(&olky) + if len(opy) > 0 { + for _, my := range opy { + ok, dErr := AddExDataToCenter(cveData, cveRef, my.Repo, openeulerNum, goe, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + return ok, dErr + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + return false, mErr + } + } else if organizationID == 3 { + ms := models.MindSporeYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + msy, mErr := models.GetMindSporeYamlAll(&ms) + if len(msy) > 0 { + for _, my := range msy { + ok, dErr := AddExDataToCenter(cveData, cveRef, my.Repo, openeulerNum, goe, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + return ok, dErr + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + return false, mErr + } + } else if organizationID == 2 { + opy := models.OpenGussYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + ogy, oErr := models.GetOpengaussYamlAll(&opy) + if len(ogy) > 0 { + for _, gy := range ogy { + ok, dErr := AddExDataToCenter(cveData, cveRef, gy.Repo, openeulerNum, goe, organizationID) + if !ok { + logs.Error("openGauss, dErr: ", dErr) + return ok, dErr + } + } + } else { + logs.Error("openGauss, oErr: ", oErr) + return false, oErr + } + } else { + ok, dErr := AddExDataToCenter(cveData, cveRef, cveData.PackName, openeulerNum, goe, organizationID) + if !ok { + logs.Error("openEuler, dErr: ", dErr) + return ok, dErr + } + } + return true, nil +} + +func AddExDataToCenter(cveData models.OriginExcel, + cveRef, repoName string, openeulerNum int, + goe models.GitPackageInfo, organizationID int8) (bool, error) { + CveRes, err := models.QueryCveByNum(cveData.CveNum, repoName, organizationID) + if err { + CveRes.RepoName = cveData.PackName + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, CveRes.CveVersion) + cveData.CveVersion = retVersion + lockx.Lock() + ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe, organizationID) + lockx.Unlock() + if !ok { + logs.Error("AddExDataToCenter, UpdateExcelCveGroups, cveData: ", cveData, ", err: ", err) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } else { + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, "") + cveData.CveVersion = retVersion + lockx.Lock() + ok, err := InsertCveExcelGroups(cveData, cveRef, repoName, openeulerNum, goe, organizationID) + lockx.Unlock() + if !ok { + logs.Error("AddExDataToCenter, InsertCveExcelGroups, cveData: ", cveData, ", err: ", err) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } + return true, nil +} + +func GetCveOriginData(prcnum, days, openeulernum, credibilityLevel int, cveRef string) (bool, error) { + defer common.Catchs() + count := 0 + beforeTime := common.GetBeforeTime(days) + for { + osx, num, err := models.QueryOriginCve(beforeTime, prcnum, credibilityLevel) + if err != nil || num == 0 { + logs.Info("There is currently no cve raw data processing, err: ", err, + ", Processing time range > beforetime: ", beforeTime, + "curtime: ", common.GetCurTime()) + break + } + ch := make(chan int, len(osx)) + for i, cveOrg := range osx { + count = count + 1 + // Add mutex + lockOk := models.LockUpdateOriginStatus(common.GetCurTime(), cveOrg.PackName, + cveOrg.Version, cveOrg.CveId, cveOrg.Status, 15) + if !lockOk { + logs.Error("Current data is being processed: PackName: ", + cveOrg.PackName, cveOrg.Version, cveOrg.CveId) + ch <- i + continue + } + go func(idx int, cveData models.OriginUpstream, locOpenEulernum int, locCveRef string) { + ok, err := GenCveVuler(cveData, locCveRef, locOpenEulernum) + if !ok { + logs.Error("GenCveVuler, cveData: ", cveData, "Processing failed, err: ", err) + } + ch <- idx + }(i, cveOrg, openeulernum, cveRef) + } + for i := 0; i < cap(ch); i++ { + <-ch + } + close(ch) + } + return true, nil +} + +func GetCveOriginExcelData(prcnum, days, openeulerNum int, cveRef string) (bool, error) { + defer common.Catchs() + manYears, ok := beego.AppConfig.Int("cve::cve_number_m") + if ok != nil { + logs.Error("config cve::manYears error:", ok) + return false, ok + } + count := 0 + beforeTime := common.GetBeforeTime(days) + for { + osx, num, err := models.QueryOriginCveExcel(beforeTime, prcnum) + if err != nil || num == 0 { + logs.Info("There is currently no cve raw data processing imported by excel, err: ", err, + ", beforetime: ", beforeTime, + "curtime: ", common.GetCurTime()) + break + } + logs.Info("Excel has a total of: ", num, "Articles of data need to be processed,", osx) + ch := make(chan int, len(osx)) + for i, cveOrg := range osx { + count = count + 1 + logs.Info("The current normal analysis: ", count, "Pieces of data, i:", i, ", cvenum: ", cveOrg.CveNum) + // add mutex + lockOk := models.LockUpdateOriginExcelStatus(common.GetCurTime(), cveOrg.CveId, 15) + if !lockOk { + logs.Error("Current data is being processed, cveNum: ", cveOrg.CveNum, ",cveId: ", cveOrg.CveId) + ch <- i + continue + } + go func(idx, manYears, locOpeneulerNum int, cveData models.OriginExcel, locCveRef string) { + ok, err := SyncCveVuler(cveData, locCveRef, openeulerNum, manYears) + if !ok { + logs.Error("SyncCveVuler, err: ", err, "cveData: ", cveData) + } + ch <- idx + }(i, manYears, openeulerNum, cveOrg, cveRef) + } + for i := 0; i < cap(ch); i++ { + <-ch + } + close(ch) + } + return true, nil +} + +func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, cveRef string, openeulernum int, + cveStatus int8, goe models.GitPackageInfo, owner string) (bool, error) { + var vul models.VulnCenter + vul.CveNum = cveData.CveNumber + specCharList := []string{"

", "\n", "\r", "\t"} + briefIntroduction := RemoveSubstring(lop.BriefIntroduction, specCharList) + vul.Description = briefIntroduction + vul.Status = cveStatus + vul.OrganizationID = cveData.OrganizationID + accessToken := "" + owner, accessToken = common.GetOwnerAndToken(vul.OrganizationID) + retVersion := AddCveVersion(vul.OrganizationID, + strings.Split(RemoveSubstring(lop.Version, specCharList), ","), "") + vul.CveVersion = retVersion + vul.RepairTime = "" + vul.PackName = cveData.RepoPath + vul.RepoName = lop.Components + vul.CveUrl = cveRef + cveData.CveNumber + vul.IsExport = 0 + vul.DataSource = 4 + v2, cvsError := strconv.ParseFloat(lop.CvsScore, 64) + if cvsError != nil { + vul.CveLevel = "Critical" + } + if len(briefIntroduction) < 2 || v2 <= 0 || len(lop.CvsVector) < 1 && + cveData.IssueState != "已拒绝" && cveData.IssueState != "已挂起" { + vul.Status = 1 + } + vul.CveLevel = models.OpenEulerScoreProc(v2) + var sec models.SecurityNotice + sec.CveNum = cveData.CveNumber + sec.InfluenceComponent = cveData.RepoPath + sec.Status = 0 + sec.AffectStatus = "Fixed" + sec.Summary = cveData.RepoPath + " security update" + if len(goe.Decription) < 1 { + goe.Decription = goe.Summary + } + sec.Description = RemoveSubstring(goe.Decription, specCharList) + "\n\n" + "Security Fix(es):" + "\n\n" + + RemoveSubstring(lop.BriefIntroduction, specCharList) + if sec.Description != "" && len(sec.Description) > 1 { + sec.Description = deleteTailBlank(sec.Description) + "(" + vul.CveNum + ")" + } else { + sec.Description += "(" + vul.CveNum + ")" + } + opScoreLeve := "" + opScore, opError := strconv.ParseFloat(lop.OpScore, 64) + if opError != nil { + opScoreLeve = "Critical" + } else { + opScoreLeve = models.OpenEulerScoreProc(opScore) + } + sec.ReferenceLink = cveRef + cveData.CveNumber + affectBrands := "" + if lop.InfVersion != "" && len(lop.InfVersion) > 1 { + affectBrands = AddAffectBrands(lop.InfVersion) + } else { + affectBrands = lop.InfProduct + } + sec.Introduction = "An update for " + cveData.RepoPath + " is now available for " + affectBrands + "." + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + sec.AffectProduct = affectBrands + var sc models.Score + sc.CveNum = cveData.CveNumber + nVDScore, nvdError := strconv.ParseFloat(lop.CvsScore, 64) + if nvdError == nil { + sc.NVDScore = nVDScore + } + openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) + if openError == nil { + sc.OpenEulerScore = openEulerScore + } + sc.NvectorVule = lop.CvsVector + sc.OvectorVule = lop.OpVector + sc.Nstatus = 0 + sc.Ostatus = 0 + sc.ScoreType = lop.ScoreType + if sc.ScoreType == "v3" { + vectorN := models.VectorValue{} + ok := vectorN.VctToVectorValue(lop.CvsVector) + if ok { + sc.NattackVector = vectorN.AttackVector + sc.NattackComplexity = vectorN.AttackComplexity + sc.NprivilegeRequired = vectorN.PrivilegesRequired + sc.NuserInteraction = vectorN.UserInteraction + sc.Nscope = vectorN.Scope + sc.Nconfidentiality = vectorN.Confidentiality + sc.Nintegrity = vectorN.Integrity + sc.Navailability = vectorN.Availability + } + vectorO := models.VectorValue{} + vectOk := vectorO.VctToVectorValue(lop.OpVector) + if vectOk { + sc.OattackVector = vectorO.AttackVector + sc.OattackComplexity = vectorO.AttackComplexity + sc.OprivilegeRequired = vectorO.PrivilegesRequired + sc.OuserInteraction = vectorO.UserInteraction + sc.Oscope = vectorO.Scope + sc.Oconfidentiality = vectorO.Confidentiality + sc.Ointegrity = vectorO.Integrity + sc.Oavailability = vectorO.Availability + } + } + if sc.ScoreType == "v2" { + vectorN2 := models.VectorValueV2{} + ok := vectorN2.VctToVectorValue(lop.CvsVector) + if ok { + sc.NaccessVector = vectorN2.AccessVector + sc.NaccessComplexity = vectorN2.AccessComplexity + sc.Navailability = vectorN2.Availability + sc.Nconfidentiality = vectorN2.Confidentiality + sc.Nintegrity = vectorN2.IntegrityImpact + sc.Nauthentication = vectorN2.Authentication + } + vectorO2 := models.VectorValueV2{} + vectOk := vectorO2.VctToVectorValue(lop.OpVector) + if vectOk { + sc.OaccessVector = vectorO2.AccessVector + sc.OaccessComplexity = vectorO2.AccessComplexity + sc.Oavailability = vectorO2.Availability + sc.Oconfidentiality = vectorO2.Confidentiality + sc.Ointegrity = vectorO2.IntegrityImpact + sc.Oauthentication = vectorO2.Authentication + } + } + var opensa models.OpenEulerSA + var OpenNumData int + //var os models.OpenSaId + osx, opError := models.QueryOpenSaLastId() + if opError == nil { + OpenNumList := strings.Split(osx.OpenEulerSANum, "-") + OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) + if err == nil { + OpenNum += 10 + } else { + OpenNum = openeulernum + } + OpenNumData = OpenNum + } else { + OpenNumData = openeulernum + } + OpenEulerSANum := "openEuler-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.Itoa(int(OpenNumData)) + opensa.OpenEulerSANum = OpenEulerSANum + var scorecode models.ScoreRecord + scorecode.NVDScore = nVDScore + scorecode.NvectorVule = lop.CvsVector + scorecode.Status = 0 + cveid, cveError := models.CreateCveRelat(&vul, &sec, &sc, &opensa, &scorecode) + if cveError != nil || cveid <= 0 { + logs.Error("insert (&CveRes, &scoreRes, &sec) failed CveNum:", cveData.CveNumber) + return false, errors.New("数据插入失败, 暂时不处理") + } + vul.CveId = cveid + cveCenter := models.VulnCenter{CveId: vul.CveId, CveNum: vul.CveNum, PackName: cveData.RepoPath} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num", "pack_name") + if cveErr != nil { + logs.Error(cveErr) + } + var issueTemp models.IssueTemplate + issueTemp.CveId = cveid + issueTemp.CveNum = vul.CveNum + issueTemp.CveNum = cveData.CveNumber + issueTemp.OwnedComponent = lop.Components + issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) + issueTemp.NVDScore = nVDScore + issueTemp.OpenEulerScore = openEulerScore + issueTemp.NVDVector = lop.CvsVector + issueTemp.OpenEulerVector = lop.OpVector + issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, specCharList) + issueTemp.CveAnalysis = lop.Influences + issueTemp.PrincipleAnalysis = lop.Principle + issueTemp.AffectedVersion = lop.InfVersion + issueTemp.AbiVersion = lop.AbiVersion + issueTemp.Solution = lop.AvoidScheme + issueTemp.IssueId = cveData.IssueId + issueTemp.IssueNum = cveData.Number + issueTemp.IssueLabel = beego.AppConfig.String("labelUnFix") + issueTemp.StatusName = cveData.State + issueTemp.Assignee = cveData.IssueAssignee + UpdateIssueState(cveData.State, cveData.IssueState, &issueTemp, 3) + if issueTemp.Status == 3 { + vul.IsExport = 3 + cveCenter.IsExport = 3 + } + issueTemp.Owner = owner + issueTemp.Repo = cveData.RepoPath + if cveData.Title != "" { + issueTemp.Title = cveData.Title + } + update := models.UpdateVulnCenter(&cveCenter, "is_export") + if !update { + logs.Error("update vulnCenter fail ") + } + issueTemp.IssueType = CIssueType + issueTemp.CveLevel = vul.CveLevel + issueTemp.MtAuditFlag = 1 + issueTemp.SaAuditFlag = 1 + issTempId, issError := models.CreateIssueTemplate(&issueTemp) + if issError != nil { + logs.Error("Failed to create data for issue template, "+ + "cveNum: ", cveData.CveNumber, ",err: ", issError) + return false, issError + } else { + logs.Info("The data of the issue template was successfully created, "+ + "issTempId: ", issTempId, ",cveNum: ", cveData.CveNumber) + } + hookError := CreateDepositHooks(accessToken, owner, cveData.RepoPath, vul, cveData.Number, cveData.IssueId) + if hookError != nil { + logs.Error(hookError) + } + return true, nil +} + +func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, cveRef string, openeulernum int, + cveStatus int8, goe models.GitPackageInfo, vul models.VulnCenter, owner string) (bool, error) { + //var vul models.VulnCenter + if vul.CveNum == "" || len(vul.CveNum) < 1 { + vul.CveNum = cveData.CveNumber + } + vul.Status = cveStatus + specCharList := []string{"

", "\n", "\r", "\t"} + briefIntroduction := RemoveSubstring(lop.BriefIntroduction, specCharList) + if (vul.Description == "" || len(vul.Description) < 1) && len(briefIntroduction) > 1 { + vul.Description = briefIntroduction + } + retVersion := AddCveVersion(cveData.OrganizationID, + strings.Split(RemoveSubstring(lop.Version, specCharList), ","), vul.CveVersion) + vul.CveVersion = retVersion + if vul.PackName == "" || len(vul.PackName) < 1 { + vul.PackName = cveData.RepoPath + } + if vul.CveUrl == "" || len(vul.CveUrl) < 1 { + vul.CveUrl = cveRef + cveData.CveNumber + } + vul.RepoName = lop.Components + vul.OrganizationID = cveData.OrganizationID + accessToken := "" + owner, accessToken = common.GetOwnerAndToken(vul.OrganizationID) + var sec models.SecurityNotice + sec.CveId = vul.CveId + sec.CveNum = vul.CveNum + sec.InfluenceComponent = vul.PackName + sec.Status = 0 + sec.AffectStatus = "Fixed" + sec.Summary = vul.PackName + " security update" + if len(goe.Decription) < 1 { + goe.Decription = goe.Summary + } + sec.Description = RemoveSubstring(goe.Decription, specCharList) + "\n\n" + "Security Fix(es):" + "\n\n" + vul.Description + if sec.Description != "" && len(sec.Description) > 1 { + sec.Description = deleteTailBlank(sec.Description) + "(" + vul.CveNum + ")" + } else { + sec.Description += "(" + vul.CveNum + ")" + } + sec.ReferenceLink = vul.CveUrl + var issueTemps models.IssueTemplate + issueTemps.CveId = vul.CveId + issueTemps.CveNum = vul.CveNum + models.GetIssueTemplateByColName(&issueTemps, "cve_id", "cve_num") + opScoreLeve := "" + opScore := float64(0) + if issueTemps.OpenEulerScore > 0 { + opScore = issueTemps.OpenEulerScore + opScoreLeve = models.OpenEulerScoreProc(opScore) + } else { + opScore, opError := strconv.ParseFloat(lop.OpScore, 64) + if opError != nil { + opScoreLeve = "Critical" + } else { + opScoreLeve = models.OpenEulerScoreProc(opScore) + } + } + if issueTemps.NVDVector != "" && len(issueTemps.NVDVector) > 1 && len(lop.CvsVector) < 2 { + lop.CvsVector = issueTemps.NVDVector + } + if issueTemps.OpenEulerVector != "" && len(issueTemps.OpenEulerVector) > 1 && len(lop.OpVector) < 2 { + lop.OpVector = issueTemps.OpenEulerVector + } + vul.CveLevel = opScoreLeve + affectBrands := "" + if lop.InfVersion != "" && len(lop.InfVersion) > 2 { + affectBrands = AddAffectBrands(lop.InfVersion) + } else { + if issueTemps.AffectedVersion != "" && len(issueTemps.AffectedVersion) > 1 { + affectBrands = AddAffectBrands(issueTemps.AffectedVersion) + } + } + sec.Introduction = "An update for " + vul.PackName + " is now available for " + affectBrands + "." + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + sec.AffectProduct = affectBrands + nVDScore := 0.0 + if issueTemps.NVDScore > 0 { + nVDScore = issueTemps.NVDScore + } + //var err error + var score models.Score + score.CveId = vul.CveId + var scOk bool + score, scOk = models.QueryScoreByCveId(vul.CveId) + if !scOk || score.Id == 0 { + //var sc models.Score + score.CveNum = vul.CveNum + var nvdError error + if issueTemps.NVDScore > 0 { + score.NVDScore = issueTemps.NVDScore + } else { + nVDScore, nvdError = strconv.ParseFloat(lop.CvsScore, 64) + if nvdError == nil && nVDScore > 0 { + score.NVDScore = nVDScore + } + } + if issueTemps.OpenEulerScore > 0 { + score.OpenEulerScore = issueTemps.OpenEulerScore + } else { + openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) + if openError == nil && openEulerScore > 0 { + score.OpenEulerScore = openEulerScore + } + } + score.NvectorVule = lop.CvsVector + score.OvectorVule = lop.OpVector + score.Nstatus = 0 + score.Ostatus = 0 + score.ScoreType = lop.ScoreType + if score.ScoreType == "v3" { + vectorN := models.VectorValue{} + ok := vectorN.VctToVectorValue(lop.CvsVector) + if ok { + score.NattackVector = vectorN.AttackVector + score.NattackComplexity = vectorN.AttackComplexity + score.NprivilegeRequired = vectorN.PrivilegesRequired + score.NuserInteraction = vectorN.UserInteraction + score.Nscope = vectorN.Scope + score.Nconfidentiality = vectorN.Confidentiality + score.Nintegrity = vectorN.Integrity + score.Navailability = vectorN.Availability + } + vectorO := models.VectorValue{} + vectOk := vectorO.VctToVectorValue(lop.OpVector) + if vectOk { + score.OattackVector = vectorO.AttackVector + score.OattackComplexity = vectorO.AttackComplexity + score.OprivilegeRequired = vectorO.PrivilegesRequired + score.OuserInteraction = vectorO.UserInteraction + score.Oscope = vectorO.Scope + score.Oconfidentiality = vectorO.Confidentiality + score.Ointegrity = vectorO.Integrity + score.Oavailability = vectorO.Availability + } + } + if score.ScoreType == "v2" { + vectorN2 := models.VectorValueV2{} + ok := vectorN2.VctToVectorValue(lop.CvsVector) + if ok { + score.NaccessVector = vectorN2.AccessVector + score.NaccessComplexity = vectorN2.AccessComplexity + score.Navailability = vectorN2.Availability + score.Nconfidentiality = vectorN2.Confidentiality + score.Nintegrity = vectorN2.IntegrityImpact + score.Nauthentication = vectorN2.Authentication + } + vectorO2 := models.VectorValueV2{} + vectOk := vectorO2.VctToVectorValue(lop.OpVector) + if vectOk { + score.OaccessVector = vectorO2.AccessVector + score.OaccessComplexity = vectorO2.AccessComplexity + score.Oavailability = vectorO2.Availability + score.Oconfidentiality = vectorO2.Confidentiality + score.Ointegrity = vectorO2.IntegrityImpact + score.Oauthentication = vectorO2.Authentication + } + } + } else { + v2 := score.NVDScore + vul.CveLevel = models.OpenEulerScoreProc(v2) + score.CveNum = vul.CveNum + score.Nstatus = 2 + var nvdError error + if issueTemps.NVDScore > 0 { + score.NVDScore = issueTemps.NVDScore + score.Nstatus = 3 + } else { + nVDScore, nvdError = strconv.ParseFloat(lop.CvsScore, 64) + if nvdError == nil && nVDScore > 0 { + if score.NVDScore != nVDScore { + score.Nstatus = 3 + score.NVDScore = nVDScore + } + } + } + if issueTemps.OpenEulerScore > 0 { + score.OpenEulerScore = issueTemps.OpenEulerScore + score.Ostatus = 3 + } else { + openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) + if openError == nil && openEulerScore > 0 { + score.OpenEulerScore = openEulerScore + score.Ostatus = 3 + } + } + score.OvectorVule = lop.OpVector + score.NvectorVule = lop.CvsVector + score.ScoreType = lop.ScoreType + if score.ScoreType == "v3" { + vectorO := models.VectorValue{} + okx := vectorO.VctToVectorValue(lop.OpVector) + if okx { + score.OattackVector = vectorO.AttackVector + score.OattackComplexity = vectorO.AttackComplexity + score.OprivilegeRequired = vectorO.PrivilegesRequired + score.OuserInteraction = vectorO.UserInteraction + score.Oscope = vectorO.Scope + score.Oconfidentiality = vectorO.Confidentiality + score.Ointegrity = vectorO.Integrity + score.Oavailability = vectorO.Availability + } + } + if score.ScoreType == "v2" { + vectorO2 := models.VectorValueV2{} + vectOk := vectorO2.VctToVectorValue(lop.OpVector) + if vectOk { + score.OaccessVector = vectorO2.AccessVector + score.OaccessComplexity = vectorO2.AccessComplexity + score.Oavailability = vectorO2.Availability + score.Oconfidentiality = vectorO2.Confidentiality + score.Ointegrity = vectorO2.IntegrityImpact + score.Oauthentication = vectorO2.Authentication + } + } + } + if vul.Status == 0 || vul.Status == 1 || vul.Status == 3 { + hookError := CreateDepositHooks(accessToken, owner, vul.PackName, vul, cveData.Number, cveData.IssueId) + logs.Info(hookError) + } + if vul.Status == 3 { + vul.Status = 2 + } + if vul.Status == 4 { + vul.Status = 1 + } + osa, osaError := models.QueryOpenSaById(vul.CveId) + if osaError == nil && osa.OpenId > 0 { + sec.OpenId = osa.OpenId + score.OpenId = osa.OpenId + } else { + var opensa models.OpenEulerSA + var OpenNumData int + //var os models.OpenSaId + osx, opError := models.QueryOpenSaLastId() + if opError == nil { + OpenNumList := strings.Split(osx.OpenEulerSANum, "-") + OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) + if err == nil { + OpenNum += 1 + } else { + OpenNum = openeulernum + } + OpenNumData = OpenNum + } else { + OpenNumData = openeulernum + } + OpenEulerSANum := "openEuler-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.Itoa(int(OpenNumData)) + opensa.OpenEulerSANum = OpenEulerSANum + opensa.CveId = vul.CveId + openId, openError := models.CreateOpenEulerSA(&opensa) + if openError != nil && openId == 0 { + logs.Error("生成openSA失败，openSA: ", opensa, ", error: ", openError) + return false, errors.New("数据插入失败, 暂时不处理") + } + sec.OpenId = openId + score.OpenId = openId + } + cveError := models.UpdateCveRelat(&vul, &sec, &score) + if cveError != nil || vul.CveId == 0 { + logs.Error("UpdateCveRelat(&vul, &sec, &sc) failed CveNum:", cveData.CveNumber, ",err:", cveError) + return false, errors.New("数据插入失败, 暂时不处理") + } + if scOk || score.Id > 0 { + if score.NVDScore != nVDScore { + var scorecode models.ScoreRecord + scorecode.NVDScore = score.NVDScore + scorecode.NvectorVule = score.NvectorVule + scorecode.Status = 0 + scorecode.CveId = vul.CveId + scRecordId, scError := models.InsertScoreRecord(&scorecode) + logs.Info(scRecordId, scError) + } + } + cveCenter := models.VulnCenter{CveId: vul.CveId, CveNum: vul.CveNum} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if cveErr != nil { + logs.Error(cveErr) + } + tmpNVDScore := float64(0) + tmpNvdError := error(nil) + if len(lop.CvsScore) > 0 { + tmpNVDScore, tmpNvdError = strconv.ParseFloat(lop.CvsScore, 64) + if tmpNvdError != nil { + logs.Error("tmpNvdError: ", tmpNvdError) + } + } + var issueTemp models.IssueTemplate + issueTemp.CveId = vul.CveId + issueTemp.CveNum = vul.CveNum + templateErr := models.GetIssueTemplateByColName(&issueTemp, "cve_id", "cve_num") + if templateErr == nil && issueTemp.TemplateId > 0 { + //issueTemp.CveNum = cveData.CveNumber + issueTemp.OwnedComponent = lop.Components + if (len(briefIntroduction) < 2 || tmpNVDScore <= 0 || len(lop.CvsVector) < 1) && issueTemp.Status < 3 && + cveData.IssueState != "已拒绝" && cveData.IssueState != "已挂起" { + cveCenter.Status = 1 + } + //issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) + openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) + if openError == nil && openEulerScore > 0 { + issueTemp.OpenEulerScore = openEulerScore + } + if len(lop.CvsVector) > 1 { + issueTemp.NVDVector = lop.CvsVector + } + nvdScore, nvdError := strconv.ParseFloat(lop.CvsScore, 64) + if nvdError == nil && nvdScore > 0 { + issueTemp.NVDScore = nvdScore + } + if lop.OpVector != "" && len(lop.OpVector) > 1 { + issueTemp.OpenEulerVector = lop.OpVector + } + if len(lop.BriefIntroduction) > 2 { + issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, specCharList) + } + if lop.Influences != "" && len(lop.Influences) > 1 { + issueTemp.CveAnalysis = lop.Influences + } + issueTemp.PrincipleAnalysis = lop.Principle + if paraAffectBrandBool(lop.InfVersion) { + issueTemp.AffectedVersion = lop.InfVersion + } + if ParaAffectAbiBool(lop.AbiVersion) { + issueTemp.AbiVersion = lop.AbiVersion + } + issueTemp.Solution = lop.AvoidScheme + statusName := issueTemp.StatusName + issueTemp.Repo = cveData.RepoPath + if issueTemp.IssueId != cveData.IssueId || issueTemp.IssueNum != cveData.Number { + tmpIssueErr, tmpIssueBody := GetGiteeIssue(accessToken, owner, vul.PackName, issueTemp.IssueNum) + cveIssueErr, cveIssueBody := GetGiteeIssue(accessToken, owner, vul.PackName, cveData.Number) + if tmpIssueErr == nil && cveIssueErr == nil { + logs.Info("GetGiteeIssue, Issue has been created,issueBody:", tmpIssueBody, cveIssueBody) + if issueTemp.Status > 3 { + issueTemp.IssueId = cveData.IssueId + issueTemp.IssueNum = cveData.Number + issueTemp.StatusName = cveData.State + statusName = cveData.State + issueTemp.Assignee = cveData.IssueAssignee + } + } else if tmpIssueErr == nil && cveIssueErr != nil { + logs.Info("The data does not change", issueTemp.IssueNum) + } else if tmpIssueErr != nil && cveIssueErr == nil { + issueTemp.IssueId = cveData.IssueId + issueTemp.IssueNum = cveData.Number + issueTemp.StatusName = cveData.State + statusName = cveData.State + issueTemp.Assignee = cveData.IssueAssignee + } + } else { + issueTemp.StatusName = cveData.State + statusName = cveData.State + } + UpdateIssueState(statusName, cveData.IssueState, &issueTemp, 1) + issueTemp.Owner = owner + if cveData.Title != "" { + issueTemp.Title = cveData.Title + } + issueTemp.IssueType = CIssueType + issueTemp.CveLevel = vul.CveLevel + } else { + issueTemp.CveNum = cveData.CveNumber + issueTemp.OwnedComponent = lop.Components + issueTemp.Repo = cveData.RepoPath + issueTemp.Owner = owner + issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) + issueTemp.NVDScore = nVDScore + openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) + if openError == nil { + issueTemp.OpenEulerScore = openEulerScore + } + issueTemp.NVDVector = lop.CvsVector + nvdScore, nvdError := strconv.ParseFloat(lop.CvsScore, 64) + if nvdError == nil { + issueTemp.NVDScore = nvdScore + } + if len(briefIntroduction) < 2 || tmpNVDScore <= 0 || len(lop.CvsVector) < 1 && + cveData.IssueState != "已拒绝" && cveData.IssueState != "已挂起" { + cveCenter.Status = 1 + } + issueTemp.NVDVector = lop.CvsVector + issueTemp.OpenEulerVector = lop.OpVector + issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, specCharList) + issueTemp.CveAnalysis = RemoveSubstring(lop.Influences, specCharList) + issueTemp.PrincipleAnalysis = lop.Principle + issueTemp.AffectedVersion = lop.InfVersion + issueTemp.AbiVersion = lop.AbiVersion + issueTemp.Solution = lop.AvoidScheme + issueTemp.IssueId = cveData.IssueId + issueTemp.IssueNum = cveData.Number + issueTemp.StatusName = cveData.State + issueTemp.IssueLabel = beego.AppConfig.String("labelUnFix") + issueTemp.Assignee = cveData.IssueAssignee + UpdateIssueState(cveData.State, cveData.IssueState, &issueTemp, 2) + if cveData.Title != "" { + issueTemp.Title = cveData.Title + } + issueTemp.IssueType = CIssueType + issueTemp.CveLevel = vul.CveLevel + } + if issueTemp.Status == 3 { + cveCenter.IsExport = 3 + } + update := models.UpdateVulnCenter(&cveCenter, "is_export", "cve_status") + if !update { + logs.Error("update vulnCenter fail ") + } + // Store issue data + issTempId, issError := models.CreateIssueTemplate(&issueTemp) + if issError != nil { + logs.Error("Failed to create data for issue template, "+ + "cveNum: ", cveData.CveNumber, ",err: ", issError) + return false, issError + } else { + logs.Info("The data of the issue template was successfully created, "+ + "issTempId: ", issTempId, ",cveNum: ", cveData.CveNumber) + } + return true, nil +} + +func UpdateIssueState(statusName, issueState string, issueTemp *models.IssueTemplate, flag int) { + if (strings.ToLower(statusName) == "open" || statusName == "待办的" || statusName == "开启的" || + issueState == "待办的" || issueState == "新建") && issueState != "已挂起" { + issueTemp.Status = 1 + issueTemp.IssueStatus = 1 + issueTemp.StatusName = "open" + } else if strings.ToLower(statusName) == "progressing" || + strings.ToLower(statusName) == "started" || statusName == "进行中" || issueState == "进行中" { + issueTemp.Status = 2 + issueTemp.IssueStatus = 3 + issueTemp.StatusName = "progressing" + } else if strings.ToLower(statusName) == "closed" || statusName == "已完成" || issueState == "已完成" { + issueTemp.Status = 3 + issueTemp.IssueStatus = 2 + issueTemp.MtAuditFlag = 1 + if flag == 2 { + issueTemp.IssueLabel = beego.AppConfig.String("labelFixed") + issueTemp.MtAuditFlag = 1 + issueTemp.SaAuditFlag = 1 + } + if flag == 3 { + issueTemp.IssueLabel = beego.AppConfig.String("labelFixed") + } + } else if strings.ToLower(statusName) == "rejected" || statusName == "已拒绝" || + issueState == "已拒绝" || issueState == "已取消" { + issueTemp.Status = 4 + issueTemp.IssueStatus = 6 + if flag == 2 { + issueTemp.IssueLabel = beego.AppConfig.String("labeUnaffected") + issueTemp.MtAuditFlag = 1 + issueTemp.SaAuditFlag = 1 + } + if flag == 3 { + issueTemp.IssueLabel = beego.AppConfig.String("labeUnaffected") + } + } else if strings.ToLower(statusName) == "suspended" || statusName == "已挂起" || issueState == "已挂起" { + issueTemp.Status = 5 + issueTemp.IssueStatus = 6 + if flag == 2 || flag == 3 { + issueTemp.IssueLabel = beego.AppConfig.String("labeUnaffected") + } + } else { + issueTemp.Status = 4 + issueTemp.IssueStatus = 6 + } +} + +func paraAffectBrandBool(affectedVersion string) bool { + unaffectedBranchList := []string{} + brandsGroup := strings.Split(affectedVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "受影响" || prams == "不受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } else { + brandList = strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "受影响" || prams == "不受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + } + } + if len(unaffectedBranchList) > 0 { + return true + } + return false +} + +func ParaAffectAbiBool(abiVersion string) bool { + unaffectedBranchList := make([]string, 0) + brandsGroup := strings.Split(abiVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "是" || prams == "否" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } else { + brandList = strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams == "是" || prams == "否" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + } + } + if len(unaffectedBranchList) > 0 { + return true + } + return false +} + +func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeulernum int, owner string, openFlag int8) (bool, error) { + defer common.Catchs() + lock.Lock() + hole, err := cveData.ParseToLoophole() + lock.Unlock() + organizationID := cveData.OrganizationID + if err != nil { + logs.Error("Data parsing error, err: ", err, ", cveData: ", cveData) + models.UpdateCveIssueStatusById(3, cveData.Id) + return false, err + } + if len(hole.Components) < 1 { + hole.Components = cveData.RepoPath + } + goe := models.GitPackageInfo{} + if organizationID == 1 { + goe, _ = models.QueryCveOpeneulerDetaildataByName(cveData.RepoPath, hole.Version) + if goe.DetailId == 0 { + logs.Error("Failed to get data, ", "Components: ", hole.Components, ",Version: ", hole.Version) + } + } else { + goe, _ = models.QueryCveOpeneulerDetaildataByName(hole.Components, hole.Version) + if goe.DetailId == 0 { + logs.Error("Failed to get data, ", "Components: ", hole.Components, ",Version: ", hole.Version) + } + } + if len(goe.Version) > 0 { + hole.Version = goe.Version + } + if cveData.State != "" && (strings.ToLower(cveData.State) == "rejected" || + cveData.State == "已拒绝" || cveData.IssueState == "已拒绝" || cveData.IssueState == "已挂起") { + logs.Info("Process data whose issue status is rejected：", cveData) + cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, organizationID) + if ok && cvd.CveNum != "" { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) + logs.Info(upOk, upError) + lock.Unlock() + } else { + lock.Lock() + okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, owner) + lock.Unlock() + if !okx { + logs.Error("Failed to get data, InsertIssueCveGroups， err: ", errx) + return false, err + } + } + models.UpdateCveStatusExportByNum(common.GetCurTime(), cveData.CveNumber, 2, cveData.RepoPath, hole.Version) + models.UpdateCveIssueStatusById(2, cveData.Id) + return true, nil + } else if cveData.State != "" && (strings.ToLower(cveData.State) == "closed" || cveData.State == "已完成") { + logs.Info("Process data whose issue status is closed：", cveData) + cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, organizationID) + if ok && cvd.CveNum != "" { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) + logs.Info(upOk, upError) + lock.Unlock() + } else { + lock.Lock() + okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, owner) + lock.Unlock() + if !okx { + logs.Error("Failed to get data, InsertIssueCveGroups， err: ", errx) + return false, err + } + } + models.UpdateCveStatusExportByNum(common.GetCurTime(), cveData.CveNumber, 3, cveData.RepoPath, hole.Version) + models.UpdateCveIssueStatusById(2, cveData.Id) + return true, nil + } else { + logs.Info("Process data whose issue status is open and processing：", cveData) + cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, organizationID) + if ok && len(cvd.CveNum) > 1 { + if openFlag == 1 { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 1, goe, cvd, owner) + lock.Unlock() + logs.Info(upOk, upError) + } else { + if cvd.Status == 0 { + if strings.ToLower(cveData.State) == "open" || cveData.State == "待办的" || + strings.ToLower(cveData.State) == "progressing" || cveData.State == "进行中" || + strings.ToLower(cveData.State) == "started" || cveData.State == "开启的" { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 1, goe, cvd, owner) + lock.Unlock() + logs.Info(upOk, upError) + } else { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 3, goe, cvd, owner) + lock.Unlock() + logs.Info(upOk, upError) + } + } else if cvd.Status == 1 { + if strings.ToLower(cveData.State) == "open" || cveData.State == "待办的" || + strings.ToLower(cveData.State) == "progressing" || cveData.State == "进行中" || + strings.ToLower(cveData.State) == "started" || cveData.State == "开启的" { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 4, goe, cvd, owner) + lock.Unlock() + logs.Info(upOk, upError) + } else { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) + lock.Unlock() + logs.Info(upOk, upError) + } + } else { + lock.Lock() + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) + lock.Unlock() + logs.Info(upOk, upError) + } + } + } else { + if strings.ToLower(cveData.State) == "open" || cveData.State == "待办的" || + strings.ToLower(cveData.State) == "progressing" || cveData.State == "进行中" || + strings.ToLower(cveData.State) == "started" || cveData.State == "开启的" { + lock.Lock() + okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 1, goe, owner) + lock.Unlock() + if !okx { + logs.Error("Failed to get data, InsertIssueCveGroups， err: ", errx) + return false, err + } + } else { + lock.Lock() + okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, owner) + lock.Unlock() + if !okx { + logs.Error("Failed to get data, InsertIssueCveGroups， err: ", errx) + return false, err + } + } + } + models.UpdateCveIssueStatusById(2, cveData.Id) + return true, nil + } +} + +func ProcAffectData(ite models.IssueTemplate) { + affectVerList := strings.Split(ite.AffectedVersion, ",") + if len(affectVerList) > 0 { + affectBranch := make([]string, 0) + affectMap := make(map[string]string) + for _, avl := range affectVerList { + affectList := make([]string, 0) + if strings.Index(avl, ":") != -1 { + affectList = strings.Split(avl, ":") + } else { + affectList = strings.Split(avl, "：") + } + if len(affectList) > 0 { + if _, ok := common.FindSliceEm(affectBranch, affectList[0]); !ok { + affectBranch = append(affectBranch, affectList[0]) + } + } + if len(affectMap) > 0 { + if len(affectList) > 1 { + if av, ok := affectMap[affectList[0]]; ok { + if len(av) >= len(affectList[1]) { + continue + } else { + affectMap[affectList[0]] = affectList[1] + } + } else { + affectMap[affectList[0]] = affectList[1] + } + } else if len(affectList) == 1 { + if _, ok := affectMap[affectList[0]]; ok { + continue + } else { + affectMap[affectList[0]] = "" + } + } + } else { + if len(affectList) > 1 { + affectMap[affectList[0]] = affectList[1] + } else { + if len(affectList) == 1 { + affectMap[affectList[0]] = "" + } + } + } + } + if len(affectBranch) > 0 { + affectSlice := make([]string, 0) + for _, af := range affectBranch { + if av, ok := affectMap[af]; ok { + if len(av) > 1 { + affectSlice = append(affectSlice, af+":"+av) + } else { + affectSlice = append(affectSlice, af+":") + } + } + } + if len(affectSlice) > 0 { + ite.AffectedVersion = strings.Join(affectSlice, ",") + models.UpdateIssueTemplate(&ite, "AffectedVersion") + } + } + } +} + +func ProcIssueTemplateBranch(duplicationDate int) { + defer common.Catchs() + beforeTime := common.GetBeforeTime(duplicationDate) + tmplateId := int64(0) + for { + iteList, num, err := models.QueryIssueTemplateAll(100, tmplateId, beforeTime) + if len(iteList) > 0 { + for _, ite := range iteList { + tmplateId = ite.TemplateId + if len(ite.AffectedVersion) > 1 { + ProcAffectData(ite) + } + } + } else { + logs.Error("num: ", num, ",err: ", err) + break + } + } +} + +func GetCveIssueData(prcnum, days, openeulernum int, cveRef, owner string, openFlag int8) (bool, error) { + defer common.Catchs() + count := 0 + for { + gs, num, err := models.QueryOriginCveIssue(prcnum) + if num == 0 { + logs.Info("There is currently no cveissue raw data processing, err: ", err, + ", curtime: ", common.GetCurTime()) + break + } + ch := make(chan int, len(gs)) + for i, cveOrg := range gs { + count = count + 1 + logs.Info("The current normal analysis: ", count, " Article data,i:", i, ", cvenum: ", cveOrg.IssueId) + // add mutex + lockOk := models.LockUpdateCveIssueStatus(15, cveOrg.Id) + if !lockOk { + logs.Error("Data is being processed, id: ", cveOrg.Id, ",cveOrg: ", cveOrg) + ch <- i + continue + } + if cveOrg.CveNumber == "" || len(cveOrg.CveNumber) == 0 { + logs.Error("Data is abnormal, do not handle, cveData: ", cveOrg) + models.UpdateCveIssueStatusById(3, cveOrg.Id) + ch <- i + continue + } + go func(idx, openeulernum int, cveData models.GiteOriginIssue, locCveRef, locOwner string, openFlag int8) { + ok, err := GenCveVulerByIssue(cveData, locCveRef, openeulernum, locOwner, openFlag) + if !ok { + logs.Error("GenCveVulerByIssue, cveData: ", cveData, "Processing failed, err: ", err) + models.UpdateCveIssueStatusById(3, cveData.Id) + } + ch <- idx + }(i, openeulernum, cveOrg, cveRef, owner, openFlag) + } + for i := 0; i < cap(ch); i++ { + <-ch + } + close(ch) + } + return true, nil +} + +//GetSecurityNotice Go to the CVE official website to obtain the cve data to determine +//whether the cve issue needs to be exported. +func GetCveSecurityNotice(cveNumber, packageName string) (bool, models.RespCveDetail) { + var detail models.RespCveDetail + var urlS url.URL + q := urlS.Query() + q.Add("cveId", cveNumber) + q.Add("packageName", packageName) + params := q.Encode() + secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") + req, _ := http.NewRequest("GET", secLinkConfig, nil) + req.URL = &url.URL{ + Scheme: req.URL.Scheme, + Host: req.URL.Host, + RawQuery: params, + //Path: "/api-cve/cve-security-notice-server/cvedatabase/getByCveId", + Path: "/api-cve/cve-security-notice-server/cvedatabase/getByCveIdAndPackageName", + } + req.URL.EscapedPath() + resp, err := http.Get(req.URL.String()) + if err != nil { + logs.Error("GetCveSecurityNotice, url: ", req.URL.String(), err) + return false, detail + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil || body == nil { + logs.Error("ReadAll, url: ", req.URL.String(), err) + return false, detail + } + logs.Info("GetCveSecurityNotice, body: ", string(body)) + err = json.Unmarshal(body, &detail) + if err != nil { + logs.Error("Unmarshal, url: ", req.URL.String(), err) + return false, detail + } + if detail.Result != nil && detail.Result.Id > 0 { + return true, detail + } + return false, detail +} + +//FilterCveExported Filter exportable data +func FilterCveExported() { + logs.Info("Start by removing cve data that has already released SA...") + data, err := models.GetCanExportVulnCenterData() + if err != nil { + logs.Error(err) + return + } + for _, v := range data { + go func(center models.VulnCenter) { + ewg.Add(1) + defer ewg.Done() + issueExist, _ := GetCveSecurityNotice(center.CveNum, center.PackName) + if issueExist { + dbLock.Lock() + center.IsExport = 1 + models.UpdateVulnCenter(¢er, "is_export") + dbLock.Unlock() + } else { + if center.IsExport == 2 { + dbLock.Lock() + center.IsExport = 3 + models.UpdateVulnCenter(¢er, "is_export") + dbLock.Unlock() + } + } + }(v) + } + ewg.Wait() + logs.Info("Excluding the cve data that has been released SA ends...") +} + +func GenerateExcelTask() error { + FilterCveExported() + return nil +} + +//GenerateExcelTrigger generate cve&security notice excel file by pr merge and influence package release. +func GenerateExcelTrigger(wgCrvf *sync.WaitGroup, fileName, startTime, fileCode, affectBranch, + csvDownPath, dir string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) UpdateInfoXml { + defer wgCrvf.Done() + logs.Info(affectBranch, ", GenerateExcelTrigger start...") + var updateInfoXml UpdateInfoXml + xmlFileName := beego.AppConfig.String("xml::updateinfo_path") + xmfilePath := filepath.Join(dir, affectBranch+"_"+xmlFileName) + var cvexml []CveXml + var dpdates Updates + securityNotice := make(map[string][]SecurityNoticeXml) + packRpmx := make(map[string][]PackRpm) + ReadXml(xmfilePath, &dpdates) + fileName = filepath.Join(dir, fileName) + localPath := filepath.Join(dir, affectBranch+"_release-package.CSV") + err := downloadPackageFile(localPath, csvDownPath, affectBranch) + if err != nil { + logs.Error(err) + return updateInfoXml + } + pkgList, err := ExtractPackageData(localPath) + if err != nil { + logs.Error(err) + } else { + su := time.Now().Format("2006-01-02") + snPrefix := "openEuler-" + su + snSuffix := int64(1001) + err = GenerateCveExcelByTrigger(affectBranch, fileName, snPrefix, startTime, + snSuffix, true, pkgList, cvrfFileList, + componentMap, cvfrFileMap, &cvexml, &dpdates, + securityNotice, packRpmx) + if err != nil { + logs.Error(err) + } + } + updateInfoXml = UpdateInfoXml{XmfilePath: xmfilePath, Cvexml: cvexml, Dpdates: &dpdates, + SecurityNotice: securityNotice, PackRpmx: packRpmx, FileName: fileName, AffectBranch: affectBranch} + logs.Info(affectBranch, ", GenerateExcelTrigger: end") + return updateInfoXml +} + +func downloadPackageFile(localPath, url, affectBranch string) error { + logs.Info("The csv file currently being downloaded is: ", url) + pkgLock.Lock() + err := GiteeDownCsv(localPath, affectBranch) + pkgLock.Unlock() + return err +} + +func GiteeDownCsv(localPath, branch string) (error) { + accessToken := os.Getenv("GITEE_TOKEN") + owner := "unsunghero" + repo := "obs_pkg_rpms" + path := fmt.Sprintf("latest_rpm/%v.csv", branch) + gitUrl := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/contents/%v?access_token=%v", + owner, repo, path, accessToken) + resp, err := http.Get(gitUrl) + if err != nil { + logs.Error("get error, url:", gitUrl, "error: ", err) + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil || body == nil { + logs.Error(err) + return err + } + //logs.Info("body: \n", string(body), "url: ", gitUrl) + var contents map[string]interface{} + err = json.Unmarshal(body, &contents) + if err != nil { + logs.Error(err) + return err + } + f, ferr := os.Create(localPath) + if ferr != nil { + logs.Error(ferr) + return ferr + } + fileType := contents["type"].(string) + encoding := contents["encoding"].(string) + content := contents["content"].(string) + if fileType == "file" && encoding == "base64" { + data, baseErr := base64.StdEncoding.DecodeString(content) + if baseErr == nil { + f.Write(data) + } + } else { + f.WriteString(content) + } + return nil +} diff --git a/cve-vulner-manager/taskhandler/cvrf.go b/cve-vulner-manager/taskhandler/cvrf.go new file mode 100644 index 0000000000000000000000000000000000000000..2f95231a5bd5a2656ef071beaab04c79ce4dde6c --- /dev/null +++ b/cve-vulner-manager/taskhandler/cvrf.go @@ -0,0 +1,1434 @@ +package taskhandler + +import ( + "bytes" + "compress/zlib" + "cvevulner/common" + "cvevulner/models" + "encoding/xml" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +type CvrfSa struct { + XMLName xml.Name `xml:"cvrfdoc,omitempty"` + Xmlns string `xml:"xmlns,attr"` + XmlnsCvrf string `xml:"xmlns:cvrf,attr"` + DocumentTitle *DocumentTitle `xml:"DocumentTitle,omitempty"` + DocumentType string `xml:"DocumentType"` + DocumentPublisher *DocumentPublisher `xml:"DocumentPublisher,omitempty"` + DocumentTracking *DocumentTracking `xml:"DocumentTracking,omitempty"` + DocumentNotes *DocumentNotes `xml:"DocumentNotes,omitempty"` + DocumentReferences *DocumentReferences `xml:"DocumentReferences,omitempty"` + ProductTree *ProductTree `xml:"ProductTree,omitempty"` + Vulnerability []Vulnerability `xml:"Vulnerability,omitempty"` +} + +type DocumentTitle struct { + XMLName xml.Name `xml:"DocumentTitle,omitempty"` + XmlLang string `xml:"xml:lang,attr"` + DocumentTitle string `xml:",innerxml"` +} + +type DocumentPublisher struct { + XMLName xml.Name `xml:"DocumentPublisher,omitempty"` + Type string `xml:"Type,attr"` + ContactDetails string `xml:"ContactDetails"` + IssuingAuthority string `xml:"IssuingAuthority"` +} + +type DocumentTracking struct { + XMLName xml.Name `xml:"DocumentTracking,omitempty"` + Identification *Identification `xml:"Identification,omitempty"` + Status string `xml:"Status"` + Version string `xml:"Version"` + RevisionHistory *RevisionHistory `xml:"RevisionHistory,omitempty"` + InitialReleaseDate string `xml:"InitialReleaseDate"` + CurrentReleaseDate string `xml:"CurrentReleaseDate"` + Generator *Generator `xml:"Generator,omitempty"` +} + +type Identification struct { + XMLName xml.Name `xml:"Identification,omitempty"` + Id string `xml:"ID"` +} + +type RevisionHistory struct { + XMLName xml.Name `xml:"RevisionHistory,omitempty"` + Revision []Revision `xml:"Revision,omitempty"` +} + +type Revision struct { + XMLName xml.Name `xml:"Revision,omitempty"` + Number string `xml:"Number"` + Date string `xml:"Date"` + Description string `xml:"Description"` +} + +type Generator struct { + XMLName xml.Name `xml:"Generator,omitempty"` + Engine string `xml:"Engine"` + Date string `xml:"Date"` +} + +type DocumentNotes struct { + XMLName xml.Name `xml:"DocumentNotes,omitempty"` + Note []Note `xml:"Note,omitempty"` +} + +type Note struct { + XMLName xml.Name `xml:"Note,omitempty"` + Title string `xml:"Title,attr"` + Type string `xml:"Type,attr"` + Ordinal string `xml:"Ordinal,attr"` + XmlLang string `xml:"xml:lang,attr"` + Note string `xml:",innerxml"` +} + +type DocumentReferences struct { + XMLName xml.Name `xml:"DocumentReferences,omitempty"` + CveReference []CveReference `xml:"Reference,omitempty"` +} + +type CveReference struct { + XMLName xml.Name `xml:"Reference,omitempty"` + Type string `xml:"Type,attr"` + CveUrl []CveUrl `xml:"URL,omitempty"` +} + +type CveUrl struct { + XMLName xml.Name `xml:"URL,omitempty"` + Url string `xml:",innerxml"` +} + +type ProductTree struct { + XMLName xml.Name `xml:"ProductTree,omitempty"` + Xmlns string `xml:"xmlns,attr"` + OpenEulerBranch []OpenEulerBranch `xml:"Branch,omitempty"` +} + +type OpenEulerBranch struct { + XMLName xml.Name `xml:"Branch,omitempty"` + Type string `xml:"Type,attr"` + Name string `xml:"Name,attr"` + FullProductName []FullProductName `xml:"FullProductName,omitempty"` +} + +type FullProductName struct { + XMLName xml.Name `xml:"FullProductName,omitempty"` + ProductId string `xml:"ProductID,attr"` + Cpe string `xml:"CPE,attr"` + FullProductName string `xml:",innerxml"` +} + +type Vulnerability struct { + XMLName xml.Name `xml:"Vulnerability,omitempty"` + Ordinal string `xml:"Ordinal,attr"` + Xmlns string `xml:"xmlns,attr"` + CveNotes *CveNotes `xml:"Notes,omitempty"` + ReleaseDate string `xml:"ReleaseDate"` + Cve string `xml:"CVE"` + ProductStatuses *ProductStatuses `xml:"ProductStatuses,omitempty"` + Threats *Threats `xml:"Threats,omitempty"` + CvssScoreSets *CVSSScoreSets `xml:"CVSSScoreSets,omitempty"` + Remediations *Remediations `xml:"Remediations,omitempty"` +} + +type CveNotes struct { + XMLName xml.Name `xml:"Notes,omitempty"` + CveNote *CveNote `xml:"Note,omitempty"` +} + +type CveNote struct { + XMLName xml.Name `xml:"Note,omitempty"` + Title string `xml:"Title,attr"` + Type string `xml:"Type,attr"` + Ordinal string `xml:"Ordinal,attr"` + XmlLnag string `xml:"xml:lang,attr"` + Note string `xml:",innerxml"` +} + +type ProductStatuses struct { + XMLName xml.Name `xml:"ProductStatuses,omitempty"` + Status *Status `xml:"Status,omitempty"` +} + +type Status struct { + XMLName xml.Name `xml:"Status,omitempty"` + Type string `xml:"Type,attr"` + ProductId []ProductId `xml:"ProductID,omitempty"` +} + +type ProductId struct { + XMLName xml.Name `xml:"ProductID,omitempty"` + ProductId string `xml:",innerxml"` +} + +type Threats struct { + XMLName xml.Name `xml:"Threats,omitempty"` + Threat *Threat `xml:"Threat,omitempty"` +} + +type Threat struct { + XMLName xml.Name `xml:"Threat,omitempty"` + Type string `xml:"Type,attr"` + Description string `xml:"Description"` +} + +type CVSSScoreSets struct { + XMLName xml.Name `xml:"CVSSScoreSets,omitempty"` + ScoreSet *ScoreSet `xml:"ScoreSet,omitempty"` +} + +type ScoreSet struct { + XMLName xml.Name `xml:"ScoreSet,omitempty"` + BaseScore string `xml:"BaseScore"` + Vector string `xml:"Vector"` +} + +type Remediations struct { + XMLName xml.Name `xml:"Remediations,omitempty"` + Remediation *Remediation `xml:"Remediation,omitempty"` +} + +type UnRemediations struct { + XMLName xml.Name `xml:"Remediations,omitempty"` + Remediation []UnRemediation `xml:"Remediation,omitempty"` +} + +type Remediation struct { + XMLName xml.Name `xml:"Remediation,omitempty"` + Type string `xml:"Type,attr"` + Description string `xml:"Description"` + Date string `xml:"DATE"` + Url string `xml:"URL"` +} + +type UnRemediation struct { + XMLName xml.Name `xml:"Remediation,omitempty"` + Type string `xml:"Type,attr"` + Description string `xml:"Description"` + Date string `xml:"DATE"` + ProductId string `xml:"ProductID"` +} + +type CveInfo struct { + CveNum string + Description string + ProductID string + CveLevel string + BaseScore float64 + Vector string + Summary string + OpenEulerSANumUrl string +} + +// Intermediate information +type SecurityNoticeCvrf struct { + CveNum string + Summary string + Theme string + ReferenceLink string + Description string + Introduction string + CveLevel string + BaseScore float64 +} + +type PackRpmCvrf struct { + PackName string +} + +type CveCvrf struct { + Introduction string + OpenEulerSANum string + PublicDate string + CveLevel string + OwnedComponent string + SecurityNotice map[string][]SecurityNoticeCvrf + PackRpmx map[string][]PackRpm + CveInfo CveInfo +} + +type BrachCveInfo struct { + OpenEulerSANum string + CvrfFileName string + CveNumSlice []string + OpenEulerScore []float64 +} + +type ComponentInfo struct { + OpenEulerSANum string + OwnedComponent string + CveNum []string + CveNumMap map[string]BrachCveInfo + OpenEulerScore []float64 + UpdateFlag int + CvrfFileName string +} + +type UnaffectCvrfSa struct { + XMLName xml.Name `xml:"cvrfdoc,omitempty"` + Xmlns string `xml:"xmlns,attr"` + XmlnsCvrf string `xml:"xmlns:cvrf,attr"` + Vulnerability []UnaffectVulnerability `xml:"Vulnerability,omitempty"` +} + +type UnaffectVulnerability struct { + XMLName xml.Name `xml:"Vulnerability,omitempty"` + Ordinal string `xml:"Ordinal,attr"` + Xmlns string `xml:"xmlns,attr"` + CveNotes *CveNotes `xml:"Notes,omitempty"` + Cve string `xml:"CVE"` + ProductStatuses *ProductStatuses `xml:"ProductStatuses,omitempty"` + CvssScoreSets *CVSSScoreSets `xml:"CVSSScoreSets,omitempty"` + Remediations *UnRemediations `xml:"Remediations,omitempty"` +} + +func UnaffectReadCvrfXml(filePath string, unaffectCvrfSa *UnaffectCvrfSa) error { + if filePath == "" || len(filePath) == 0 { + logs.Error("read err: ", filePath) + return errors.New("file does not exist") + } + fisExist, ferr := PathExists(filePath) + if !fisExist { + logs.Error(ferr) + return ferr + } + fd, err := os.Open(filePath) + if err != nil { + logs.Error("open file err : ", err, ",filePath: ", filePath) + return err + } + defer fd.Close() + fileContent, err := ioutil.ReadAll(fd) + if err != nil { + logs.Error("read file err : ", err, ", filePath: ", filePath) + return err + } + err = xml.Unmarshal(fileContent, unaffectCvrfSa) + if err != nil { + logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) + return err + } + return nil +} + +func WriteUnaffectCvrfXml(filePath string, unaffectCvrfsa *UnaffectCvrfSa) { + os.Remove(filePath) + xmlOutPut, outPutErr := xml.MarshalIndent(unaffectCvrfsa, "", " ") + if outPutErr == nil { + headerBytes := []byte(xml.Header) + xmlOutPutData := append(headerBytes, xmlOutPut...) + ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) + } else { + logs.Error(outPutErr) + } +} + +func ReadCvrfXml(filePath string, cvrf *CvrfSa) error { + if filePath == "" || len(filePath) == 0 { + logs.Error("read err: ", filePath) + return errors.New("file does not exist") + } + fisExist, ferr := PathExists(filePath) + if !fisExist { + logs.Error(ferr) + return ferr + } + fd, err := os.Open(filePath) + if err != nil { + logs.Error("open file err : ", err, ",filePath: ", filePath) + return err + } + defer fd.Close() + fileContent, err := ioutil.ReadAll(fd) + if err != nil { + logs.Error("read file err : ", err, ", filePath: ", filePath) + return err + } + err = xml.Unmarshal(fileContent, cvrf) + if err != nil { + logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) + return err + } + return nil +} + +func WriteCvrfXml(filePath string, cvrfsa *CvrfSa) { + os.Remove(filePath) + xmlOutPut, outPutErr := xml.MarshalIndent(cvrfsa, "", " ") + if outPutErr == nil { + headerBytes := []byte(xml.Header) + xmlOutPutData := append(headerBytes, xmlOutPut...) + ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) + } else { + logs.Error(outPutErr) + } +} + +func BuildUnaffectVulnerabilitySet(unaffectCvrfsa *UnaffectCvrfSa, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) { + vulnerability := make([]UnaffectVulnerability, 0) + vulnerabilityx := make([]UnaffectVulnerability, 0) + if len(unaffectCvrfsa.Vulnerability) > 0 { + vulnerabilityx = BuildUnaffVulnerabilitySlice(unaffectCvrfsa.Vulnerability, v, affectBranch, componentMap) + } else { + vulnerabilityx = BuildUnaffVulnerabilitySlice(vulnerability, v, affectBranch, componentMap) + } + unaffectCvrfsa.Vulnerability = vulnerabilityx +} + +func BuildUnaffVulnerabilitySlice(vulnerability []UnaffectVulnerability, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) []UnaffectVulnerability { + cpe := affectBranch + if vulnerability != nil && len(vulnerability) > 0 { + cveExist := false + for i, vl := range vulnerability { + if vl.Cve == v.CveNum && vl.Remediations != nil && len(vl.Remediations.Remediation) > 0 && + vl.Remediations.Remediation[0].Description == v.InfluenceComponent { + cpeExist := false + for _, pid := range vl.ProductStatuses.Status.ProductId { + if pid.ProductId == cpe { + cpeExist = true + break + } + } + if !cpeExist { + var productId ProductId + productId.ProductId = cpe + vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + var remediation UnRemediation + remediation.Type = "Unaffected" + remediation.Description = v.InfluenceComponent + remediation.Date = common.GetCurDate() + remediation.ProductId = cpe + vl.Remediations.Remediation = append(vl.Remediations.Remediation, remediation) + vulnerability[i] = vl + } + cveExist = true + break + } + } + if !cveExist { + vlLenth := len(vulnerability) + 1 + vulnerabilitySlice := BuildUnaffVulnerability(vlLenth, v, componentMap, cpe) + if len(vulnerabilitySlice) > 0 { + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + } + } else { + vlLenth := 1 + vulnerabilitySlice := BuildUnaffVulnerability(vlLenth, v, componentMap, cpe) + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + return vulnerability +} + +func BuildUnaffVulnerability(vlLenth int, v models.ExcelExport, + componentMap map[string]ComponentInfo, cpe string) []UnaffectVulnerability { + vulnerabilitySlice := make([]UnaffectVulnerability, 0) + var vulnerability UnaffectVulnerability + vulnerability.Xmlns = "http://www.icasi.org/CVRF/schema/vuln/1.1" + vulnerability.Ordinal = strconv.Itoa(vlLenth) + var cveNotes CveNotes + var cveNote CveNote + cveNote.Ordinal = strconv.Itoa(vlLenth) + cveNote.Type = "General" + cveNote.Title = "Vulnerability Description" + cveBrief := XmlSpecCharHand(v.CveBrief) + cveNote.Note = cveBrief + cveNote.XmlLnag = "en" + cveNotes.CveNote = &cveNote + vulnerability.CveNotes = &cveNotes + vulnerability.Cve = v.CveNum + var productStatuses ProductStatuses + var status Status + status.Type = "Unaffected" + var productId ProductId + productId.ProductId = cpe + productIdSlice := make([]ProductId, 0) + productIdSlice = append(productIdSlice, productId) + status.ProductId = productIdSlice + productStatuses.Status = &status + vulnerability.ProductStatuses = &productStatuses + var cVSSScoreSets CVSSScoreSets + var scoreSet ScoreSet + scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpenEulerScore) + scoreSet.Vector = v.OvectorVule + cVSSScoreSets.ScoreSet = &scoreSet + vulnerability.CvssScoreSets = &cVSSScoreSets + var remediations UnRemediations + remediationSlice := make([]UnRemediation, 0) + var remediation UnRemediation + remediation.Type = "Unaffected" + remediation.Description = v.InfluenceComponent + remediation.Date = common.GetCurDate() + remediation.ProductId = cpe + remediationSlice = append(remediationSlice, remediation) + remediations.Remediation = remediationSlice + vulnerability.Remediations = &remediations + vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) + return vulnerabilitySlice +} + +func BranchExist(affectBranch string, cvrfFileList map[string][]string) []string { + brancsList, keyOk := cvrfFileList[BRANCHSKEY] + if !keyOk { + localBran := make([]string, 0) + localBran = append(localBran, affectBranch) + cvrfFileList[BRANCHSKEY] = localBran + } else { + brlFlag := false + for _, brl := range brancsList { + if brl == affectBranch { + brlFlag = true + break + } + } + if !brlFlag { + brancsList = append(brancsList, affectBranch) + cvrfFileList[BRANCHSKEY] = brancsList + } + } + brancsListx, _ := cvrfFileList[BRANCHSKEY] + if len(brancsListx) > 1 { + sort.Strings(brancsListx) + } + return brancsListx +} + +func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, + introduction, topic, affectBranch string, + componentMap map[string]ComponentInfo, + branchList []string, branchFlag int) { + documentNotes := cvrfsa.DocumentNotes + if documentNotes == nil || cvrfsa.DocumentNotes.Note == nil || + len(cvrfsa.DocumentNotes.Note) == 0 { + var documentNotesx DocumentNotes + note := make([]Note, 0) + var noteSynopsis Note + noteSynopsis.Title = "Synopsis" + noteSynopsis.Type = "General" + noteSynopsis.Ordinal = "1" + noteSynopsis.XmlLang = "en" + noteSynopsis.Note = v.Summary + note = append(note, noteSynopsis) + var noteSummary Note + noteSummary.Title = "Summary" + noteSummary.Type = "General" + noteSummary.Ordinal = "2" + noteSummary.XmlLang = "en" + noteSummary.Note = introduction + note = append(note, noteSummary) + var noteDescription Note + noteDescription.Title = "Description" + noteDescription.Type = "General" + noteDescription.Ordinal = "3" + noteDescription.XmlLang = "en" + descriptionStr := strings.ReplaceAll(v.Description, "\n\n", "\r\n\r\n") + descriptionStr = XmlSpecCharHand(descriptionStr) + noteDescription.Note = descriptionStr + note = append(note, noteDescription) + var noteTopic Note + noteTopic.Title = "Topic" + noteTopic.Type = "General" + noteTopic.Ordinal = "4" + noteTopic.XmlLang = "en" + topic := strings.ReplaceAll(topic, "\n\n", "\r\n\r\n") + topic = XmlSpecCharHand(topic) + noteTopic.Note = topic + note = append(note, noteTopic) + var noteSeverity Note + noteSeverity.Title = "Severity" + noteSeverity.Type = "General" + noteSeverity.Ordinal = "5" + noteSeverity.XmlLang = "en" + cveLevel := models.OpenEulerScoreProc(v.OpenEulerScore) + noteSeverity.Note = cveLevel + note = append(note, noteSeverity) + var noteComponent Note + noteComponent.Title = "Affected Component" + noteComponent.Type = "General" + noteComponent.Ordinal = "6" + noteComponent.XmlLang = "en" + noteComponent.Note = v.InfluenceComponent + note = append(note, noteComponent) + documentNotesx.Note = note + cvrfsa.DocumentNotes = &documentNotesx + } else { + branchCount := len(branchList) + note := cvrfsa.DocumentNotes.Note + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + notex := make([]Note, 0) + for _, te := range note { + if te.Title == "Description" { + descriptionStr := strings.ReplaceAll(v.Description, "\n\n", "\r\n\r\n") + descriptionStr = XmlSpecCharHand(descriptionStr) + dSplit := strings.Split(descriptionStr, "Security Fix(es):") + if len(dSplit) > 1 { + if !strings.Contains(te.Note, dSplit[0]) { + te.Note = dSplit[0] + te.Note + } + if !strings.Contains(te.Note, dSplit[1]) { + te.Note += dSplit[1] + } + } + te.Note = te.Note + } + if te.Title == "Topic" { + vcn := "" + if branchFlag == 1 { + vcn = strings.Join(componentInfo.CveNum, ";\n") + theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) + if err == nil && len(theme) > 1 { + theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") + theme = XmlSpecCharHand(theme) + if branchCount <= 1 { + te.Note = theme + } else if branchCount == 2 { + te.Note = strings.ReplaceAll(theme, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + te.Note = strings.ReplaceAll(theme, affectBranch, reBanch) + } + } + } else { + vcn = strings.Join(componentInfo.CveNumMap[affectBranch].CveNumSlice, ";\n") + theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) + if err == nil && len(theme) > 1 { + theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") + theme = XmlSpecCharHand(theme) + te.Note = theme + } + } + } + if te.Title == "Summary" { + if branchFlag == 1 { + if branchCount <= 1 { + te.Note = v.Introduction + } else if branchCount == 2 { + te.Note = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + te.Note = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + } + } else { + te.Note = v.Introduction + } + + } + if te.Title == "Severity" { + var openEulerScoreSlice []float64 + if branchFlag == 1 { + openEulerScoreSlice = componentInfo.OpenEulerScore + } else { + openEulerScoreSlice = componentInfo.CveNumMap[affectBranch].OpenEulerScore + } + + if len(openEulerScoreSlice) > 1 { + sort.Float64s(openEulerScoreSlice) + } + if len(openEulerScoreSlice) > 0 { + cveLevel := models.OpenEulerScoreProc(openEulerScoreSlice[len(openEulerScoreSlice)-1]) + te.Note = cveLevel + } + } + notex = append(notex, te) + } + cvrfsa.DocumentNotes.Note = notex + } +} + +func BuildDocumentTitle(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, + cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, + curDate string, branchList []string, branchFlag int) { + var documt DocumentTitle + introduction := "" + topic := "" + branchCount := len(branchList) + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + localOpenEulerSANum := "" + if branchFlag == 1 { + localOpenEulerSANum = componentInfo.OpenEulerSANum + if branchCount <= 1 { + introduction = v.Introduction + topic = v.Theme + } else if branchCount == 2 { + introduction = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + topic = strings.ReplaceAll(v.Theme, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + introduction = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + topic = strings.ReplaceAll(v.Theme, affectBranch, reBanch) + } + } else { + introduction = v.Introduction + topic = v.Theme + localOpenEulerSANum = componentInfo.CveNumMap[affectBranch].OpenEulerSANum + } + if len(introduction) > 1 { + documt.DocumentTitle = introduction[:len(introduction)-1] + } else { + documt.DocumentTitle = introduction + } + documt.XmlLang = "en" + cvrfsa.DocumentTitle = &documt + cvrfsa.DocumentType = "Security Advisory" + var documentPublisher DocumentPublisher + documentPublisher.Type = "Vendor" + documentPublisher.ContactDetails = "openeuler-security@openeuler.org" + documentPublisher.IssuingAuthority = "openEuler security committee" + cvrfsa.DocumentPublisher = &documentPublisher + var documentTracking DocumentTracking + var identification Identification + identification.Id = localOpenEulerSANum + documentTracking.Identification = &identification + documentTracking.Status = "Final" + documentTracking.Version = "1.0" + var revisionHistory RevisionHistory + revision := make([]Revision, 0) + if cvrfsa.DocumentTracking != nil && + cvrfsa.DocumentTracking.RevisionHistory != nil && + len(cvrfsa.DocumentTracking.RevisionHistory.Revision) > 0 && + componentInfo.UpdateFlag == 1 { + RevisionSlice := cvrfsa.DocumentTracking.RevisionHistory.Revision + for _, rev := range RevisionSlice { + revision = append(revision, rev) + } + versionValue := float64(len(RevisionSlice))/float64(10) + float64(1) + var revisionx Revision + revisionx.Number = fmt.Sprintf("%.1f", versionValue) + revisionx.Date = v.PublicDate + revisionx.Description = "Update" + revision = append(revision, revisionx) + documentTracking.Version = revisionx.Number + } else { + var revisionx Revision + revisionx.Number = "1.0" + revisionx.Date = v.PublicDate + revisionx.Description = "Initial" + revision = append(revision, revisionx) + documentTracking.InitialReleaseDate = v.PublicDate + } + revisionHistory.Revision = revision + documentTracking.RevisionHistory = &revisionHistory + documentTracking.CurrentReleaseDate = v.PublicDate + var generator Generator + generator.Date = curDate + generator.Engine = "openEuler SA Tool V1.0" + documentTracking.Generator = &generator + cvrfsa.DocumentTracking = &documentTracking + BuilddocumentNotes(cvrfsa, v, introduction, topic, + affectBranch, componentMap, branchList, branchFlag) +} + +func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, + componentMap map[string]ComponentInfo, branchFlag int, affectBranch string) { + componentInfo, comOk := componentMap[v.InfluenceComponent] + localOpenEulerSANum := "" + secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") + localCveNum := []string{} + if branchFlag == 1 { + localOpenEulerSANum = componentInfo.OpenEulerSANum + localCveNum = componentInfo.CveNum + } else { + localOpenEulerSANum = componentInfo.CveNumMap[affectBranch].OpenEulerSANum + localCveNum = componentInfo.CveNumMap[affectBranch].CveNumSlice + } + if comOk { + if cvrfsa.DocumentReferences != nil && len(cvrfsa.DocumentReferences.CveReference) > 0 { + cveReference := cvrfsa.DocumentReferences.CveReference + cveReferencex := make([]CveReference, 0) + for _, cveRef := range cveReference { + if cveRef.Type == "openEuler CVE" { + isExist := false + for _, cUrl := range cveRef.CveUrl { + if strings.Contains(cUrl.Url, v.CveNum) { + isExist = true + } + } + if !isExist { + var cveUrl1 CveUrl + cveUrl1.Url = secLinkConfig + "/en/security/cve/detail.html?id=" + v.CveNum + cveRef.CveUrl = append(cveRef.CveUrl, cveUrl1) + } + } + if cveRef.Type == "Other" { + isExist := false + for _, cUrl := range cveRef.CveUrl { + if strings.Contains(cUrl.Url, v.CveNum) { + isExist = true + } + } + if !isExist { + var cveUrl2 CveUrl + cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + v.CveNum + cveRef.CveUrl = append(cveRef.CveUrl, cveUrl2) + } + } + cveReferencex = append(cveReferencex, cveRef) + } + cvrfsa.DocumentReferences.CveReference = cveReferencex + } else { + var documentReferences DocumentReferences + cveReferenceSlice := make([]CveReference, 0) + var cveReference0 CveReference + cveUrlSlice0 := make([]CveUrl, 0) + var cveUrl0 CveUrl + cveUrl0.Url = secLinkConfig + "/en/security/safety-bulletin/detail.html?id=" + localOpenEulerSANum + cveUrlSlice0 = append(cveUrlSlice0, cveUrl0) + cveReference0.Type = "Self" + cveReference0.CveUrl = cveUrlSlice0 + cveReferenceSlice = append(cveReferenceSlice, cveReference0) + var cveReference1 CveReference + cveUrlSlice1 := make([]CveUrl, 0) + for _, cveNum := range localCveNum { + var cveUrl1 CveUrl + cveUrl1.Url = secLinkConfig + "/en/security/cve/detail.html?id=" + cveNum + cveUrlSlice1 = append(cveUrlSlice1, cveUrl1) + } + cveReference1.Type = "openEuler CVE" + cveReference1.CveUrl = cveUrlSlice1 + cveReferenceSlice = append(cveReferenceSlice, cveReference1) + var cveReference2 CveReference + cveUrlSlice2 := make([]CveUrl, 0) + for _, cveNum := range localCveNum { + var cveUrl2 CveUrl + cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + cveNum + cveUrlSlice2 = append(cveUrlSlice2, cveUrl2) + } + cveReference2.Type = "Other" + cveReference2.CveUrl = cveUrlSlice2 + cveReferenceSlice = append(cveReferenceSlice, cveReference2) + documentReferences.CveReference = cveReferenceSlice + cvrfsa.DocumentReferences = &documentReferences + } + } +} + +func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package, + affectBranch string, branchFlag int) { + if cvrfsa.ProductTree != nil && len(cvrfsa.ProductTree.OpenEulerBranch) > 0 && branchFlag == 1 { + affectBranchListx := strings.Split(affectBranch, "-") + cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + openEulerBranchx := make([]OpenEulerBranch, 0) + for _, opBranch := range cvrfsa.ProductTree.OpenEulerBranch { + if opBranch.Type == "Product Name" && strings.Contains(affectBranch, opBranch.Name) { + fpFlag := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == affectBranch { + fpFlag = true + break + } + } + if !fpFlag { + var fullProductName0 FullProductName + fullProductName0.FullProductName = affectBranch + fullProductName0.ProductId = affectBranch + fullProductName0.Cpe = cpe + opBranch.FullProductName = append(opBranch.FullProductName, fullProductName0) + } + } + if opBranch.Type == "Package Arch" { + fullProductNameSliceaarch64 := make([]FullProductName, 0) + fullProductNameSlicenoarch := make([]FullProductName, 0) + fullProductNameSlicesrc := make([]FullProductName, 0) + fullProductNameSlicex86_64 := make([]FullProductName, 0) + if len(pkg) > 0 { + for _, pk := range pkg { + if opBranch.Name == "aarch64" { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName && fp.Cpe == cpe { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".aarch64.") { + var fullProductNameaarch64 FullProductName + fullProductNameaarch64.FullProductName = pk.PackName + fullProductNameaarch64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNameaarch64.ProductId = pk.PackName[:index] + } + fullProductNameSliceaarch64 = append(fullProductNameSliceaarch64, fullProductNameaarch64) + } + } else if opBranch.Name == "noarch" { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName && fp.Cpe == cpe { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".noarch.") { + var fullProductNamenoarch FullProductName + fullProductNamenoarch.FullProductName = pk.PackName + fullProductNamenoarch.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamenoarch.ProductId = pk.PackName[:index] + } + fullProductNameSlicenoarch = append(fullProductNameSlicenoarch, fullProductNamenoarch) + } + } else if opBranch.Name == "x86_64" { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName && fp.Cpe == cpe { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".x86_64.") { + var fullProductNamex86_64 FullProductName + fullProductNamex86_64.FullProductName = pk.PackName + fullProductNamex86_64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamex86_64.ProductId = pk.PackName[:index] + } + fullProductNameSlicex86_64 = append(fullProductNameSlicex86_64, fullProductNamex86_64) + } + } else { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName && fp.Cpe == cpe { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".src.") { + var fullProductNamesrc FullProductName + fullProductNamesrc.FullProductName = pk.PackName + fullProductNamesrc.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamesrc.ProductId = pk.PackName[:index] + } + fullProductNameSlicesrc = append(fullProductNameSlicesrc, fullProductNamesrc) + } + } + } + } + if opBranch.Name == "aarch64" { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSliceaarch64...) + } else if opBranch.Name == "noarch" { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicenoarch...) + } else if opBranch.Name == "x86_64" { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicex86_64...) + } else { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicesrc...) + } + } + openEulerBranchx = append(openEulerBranchx, opBranch) + } + cvrfsa.ProductTree.OpenEulerBranch = openEulerBranchx + } else { + var productTree ProductTree + productTree.Xmlns = "http://www.icasi.org/CVRF/schema/prod/1.1" + openEulerBranchSlice := make([]OpenEulerBranch, 0) + var openEulerBranch0 OpenEulerBranch + fullProductNameSlice0 := make([]FullProductName, 0) + var fullProductName0 FullProductName + fullProductName0.FullProductName = affectBranch + fullProductName0.ProductId = affectBranch + affectBranchListx := strings.Split(affectBranch, "-") + cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + fullProductName0.Cpe = cpe + fullProductNameSlice0 = append(fullProductNameSlice0, fullProductName0) + openEulerBranch0.FullProductName = fullProductNameSlice0 + openEulerBranch0.Type = "Product Name" + openEulerBranch0.Name = affectBranchListx[0] + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranch0) + fullProductNameSliceaarch64 := make([]FullProductName, 0) + fullProductNameSlicenoarch := make([]FullProductName, 0) + fullProductNameSlicesrc := make([]FullProductName, 0) + fullProductNameSlicex86_64 := make([]FullProductName, 0) + if len(pkg) > 0 { + for _, pk := range pkg { + if strings.Contains(pk.PackName, ".aarch64.") { + var fullProductNameaarch64 FullProductName + fullProductNameaarch64.FullProductName = pk.PackName + fullProductNameaarch64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNameaarch64.ProductId = pk.PackName[:index] + } + fullProductNameSliceaarch64 = append(fullProductNameSliceaarch64, fullProductNameaarch64) + } else if strings.Contains(pk.PackName, ".src.") { + var fullProductNamesrc FullProductName + fullProductNamesrc.FullProductName = pk.PackName + fullProductNamesrc.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamesrc.ProductId = pk.PackName[:index] + } + fullProductNameSlicesrc = append(fullProductNameSlicesrc, fullProductNamesrc) + } else if strings.Contains(pk.PackName, ".x86_64.") { + var fullProductNamex86_64 FullProductName + fullProductNamex86_64.FullProductName = pk.PackName + fullProductNamex86_64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamex86_64.ProductId = pk.PackName[:index] + } + fullProductNameSlicex86_64 = append(fullProductNameSlicex86_64, fullProductNamex86_64) + } else { + var fullProductNamenoarch FullProductName + fullProductNamenoarch.FullProductName = pk.PackName + fullProductNamenoarch.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamenoarch.ProductId = pk.PackName[:index] + } + fullProductNameSlicenoarch = append(fullProductNameSlicenoarch, fullProductNamenoarch) + } + } + } + if len(fullProductNameSliceaarch64) > 0 { + var openEulerBranchaarch64 OpenEulerBranch + openEulerBranchaarch64.Type = "Package Arch" + openEulerBranchaarch64.Name = "aarch64" + openEulerBranchaarch64.FullProductName = fullProductNameSliceaarch64 + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchaarch64) + } + if len(fullProductNameSlicenoarch) > 0 { + var openEulerBranchnoarch OpenEulerBranch + openEulerBranchnoarch.Type = "Package Arch" + openEulerBranchnoarch.Name = "noarch" + openEulerBranchnoarch.FullProductName = fullProductNameSlicenoarch + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchnoarch) + } + if len(fullProductNameSlicesrc) > 0 { + var openEulerBranchsrc OpenEulerBranch + openEulerBranchsrc.Type = "Package Arch" + openEulerBranchsrc.Name = "src" + openEulerBranchsrc.FullProductName = fullProductNameSlicesrc + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchsrc) + } + if len(fullProductNameSlicex86_64) > 0 { + var openEulerBranchx86_64 OpenEulerBranch + openEulerBranchx86_64.Type = "Package Arch" + openEulerBranchx86_64.Name = "x86_64" + openEulerBranchx86_64.FullProductName = fullProductNameSlicex86_64 + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchx86_64) + } + productTree.OpenEulerBranch = openEulerBranchSlice + cvrfsa.ProductTree = &productTree + } +} + +func BuildVulnerability(vlLenth int, v models.ExcelExport, + componentMap map[string]ComponentInfo, cpe string, branchFlag int) []Vulnerability { + vulnerabilitySlice := make([]Vulnerability, 0) + var vulnerability Vulnerability + vulnerability.Xmlns = "http://www.icasi.org/CVRF/schema/vuln/1.1" + vulnerability.Ordinal = strconv.Itoa(vlLenth) + var cveNotes CveNotes + var cveNote CveNote + cveNote.Ordinal = strconv.Itoa(vlLenth) + cveNote.Type = "General" + cveNote.Title = "Vulnerability Description" + cveBrief := XmlSpecCharHand(v.CveBrief) + cveNote.Note = cveBrief + cveNote.XmlLnag = "en" + cveNotes.CveNote = &cveNote + vulnerability.CveNotes = &cveNotes + vulnerability.ReleaseDate = v.PublicDate + vulnerability.Cve = v.CveNum + var productStatuses ProductStatuses + var status Status + status.Type = "Fixed" + var productId ProductId + productId.ProductId = cpe + productIdSlice := make([]ProductId, 0) + productIdSlice = append(productIdSlice, productId) + status.ProductId = productIdSlice + productStatuses.Status = &status + vulnerability.ProductStatuses = &productStatuses + var threats Threats + var threat Threat + threat.Type = "Impact" + threat.Description = models.OpenEulerScoreProc(v.OpenEulerScore) + threats.Threat = &threat + vulnerability.Threats = &threats + var cVSSScoreSets CVSSScoreSets + var scoreSet ScoreSet + scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpenEulerScore) + scoreSet.Vector = v.OvectorVule + cVSSScoreSets.ScoreSet = &scoreSet + vulnerability.CvssScoreSets = &cVSSScoreSets + var remediations Remediations + var remediation Remediation + remediation.Type = "Vendor Fix" + remediation.Description = v.Summary + remediation.Date = v.PublicDate + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + localOpenEulerSANum := "" + if branchFlag == 1 { + localOpenEulerSANum = componentInfo.OpenEulerSANum + } else { + localOpenEulerSANum = componentInfo.CveNumMap[cpe].OpenEulerSANum + } + secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") + remediation.Url = secLinkConfig + "/en/security/safety-bulletin/detail.html?id=" + localOpenEulerSANum + remediations.Remediation = &remediation + vulnerability.Remediations = &remediations + vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) + return vulnerabilitySlice +} + +func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo, branchFlag int) []Vulnerability { + //affectBranchListx := strings.Split(affectBranch, "-") + cpe := affectBranch + if vulnerability != nil && len(vulnerability) > 0 { + cveExist := false + for _, vl := range vulnerability { + if vl.Cve == v.CveNum && strings.Contains(vl.Remediations.Remediation.Description, v.InfluenceComponent) { + cpeExist := false + for _, pid := range vl.ProductStatuses.Status.ProductId { + if pid.ProductId == cpe { + cpeExist = true + break + } + } + if !cpeExist { + var productId ProductId + productId.ProductId = cpe + if branchFlag == 1 { + vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + } else { + productIdSlice := make([]ProductId, 0) + productIdSlice = append(productIdSlice, productId) + vl.ProductStatuses.Status.ProductId = productIdSlice + } + } + cveExist = true + break + } + } + if !cveExist { + vlLenth := len(vulnerability) + 1 + vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe, branchFlag) + if len(vulnerabilitySlice) > 0 { + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + } + } else { + vlLenth := 1 + vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe, branchFlag) + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + return vulnerability +} + +func BuildVulnerabilitySet(cvrfsa *CvrfSa, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo, branchFlag int) { + vulnerability := make([]Vulnerability, 0) + if cvrfsa.Vulnerability != nil && len(cvrfsa.Vulnerability) > 0 { + for _, vuln := range cvrfsa.Vulnerability { + vulnerability = append(vulnerability, vuln) + } + } + vulnerabilityx := BuildVulnerabilitySlice(vulnerability, v, + affectBranch, componentMap, branchFlag) + cvrfsa.Vulnerability = vulnerabilityx +} + +func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, + cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, + pkg []models.Package, branchFlag int) { + curDate := common.GetCurDate() + branchList := BranchExist(affectBranch, cvrfFileList) + cvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + cvrfsa.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + BuildDocumentTitle(cvrfsa, v, affectBranch, cvrfFileList, componentMap, curDate, branchList, branchFlag) + BuildDocumentRef(cvrfsa, v, componentMap, branchFlag, affectBranch) + BuildProductTree(cvrfsa, v, pkg, affectBranch, branchFlag) + BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap, branchFlag) +} + +func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum string, fixFlag int8) (bool, string) { + sort.Strings(cveNum) + tmpOpenEulerSANum := "" + cveStr := strings.Join(cveNum, ",") + cveStrx := cveStr + "," + OwnedComponent + fileMd5 := common.EncryptMd5(cveStrx) + var cfr models.CvrfSaRecord + cfr.Md5 = fileMd5 + tbErr := models.GetCvrfRecord(&cfr, "cur_md5") + if tbErr == nil { + tmpOpenEulerSANum = cfr.OpenEulerSANum + cfr.UpdateTime = common.GetCurTime() + cfr.Md5 = fileMd5 + cfr.OpenEulerSANum = openEulerSANum + cfr.Status = 1 + cfr.IsExport = 1 + cfr.CveNum = cveStr + cfr.PackName = OwnedComponent + cfr.AffectFlag = fixFlag + updateErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", + "OpenEulerSANum", "Status", "IsExport", "CveNum", "PackName", "AffectFlag") + if updateErr != nil { + logs.Error("UpdateCvrfRecord, ", updateErr) + } + return true, tmpOpenEulerSANum + } else { + cfr.CreateTime = common.GetCurTime() + cfr.Md5 = fileMd5 + cfr.OpenEulerSANum = openEulerSANum + cfr.Status = 1 + cfr.IsExport = 1 + cfr.CveNum = cveStr + cfr.PackName = OwnedComponent + cfr.AffectFlag = fixFlag + num, iErr := models.InsertCvrfRecord(&cfr) + if iErr != nil || num == 0 { + logs.Error("InsertCvrfRecord, ", iErr) + } + return false, tmpOpenEulerSANum + } +} + +func UpdateCvrfRecord(openeuler_sa_num string, isExport int8) { + var cfr models.CvrfSaRecord + cfr.UpdateTime = common.GetCurTime() + cfr.Status = 1 + cfr.IsExport = isExport + cfr.OpenEulerSANum = openeuler_sa_num + uErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", "Status", "IsExport") + if uErr != nil { + logs.Error("UpdateCvrfRecord, ", uErr) + } +} + +func ProcCvrfFileName(fileName string) { + var cfr models.SaFileList + cfr.FileName = fileName + tbErr := models.GetCvrfFileName(&cfr, "file_name") + if tbErr == nil { + cfr.Status = 1 + cfr.UpdateTime = common.GetCurTime() + models.UpdateCvrfFileName(&cfr, "Status", "UpdateTime") + } else { + cfr.CreateTime = common.GetCurTime() + cfr.Status = 1 + cfr.FileName = fileName + num, iErr := models.InsertCvrfFileName(&cfr) + if iErr != nil || num == 0 { + logs.Error("InsertCvrfRecord, ", iErr) + } + } +} + +func RecordCrvfInfo(fileName, filex, newText string, fixFlag int8) error { + fileBytes, err := ioutil.ReadFile(fileName) + if err != nil { + fmt.Println("ioutil.ReadFile, error : %s", err, fileName) + return err + } + fileStr := string(fileBytes) + fileMd5 := common.EncryptMd5(fileStr) + openEulerSANum := "" + if fixFlag == UNAFFECTFLAG { + openEulerSANum = fileMd5 + } else { + if len(filex) > 5 { + openEulerSANum = filex[5 : len(filex)-4] + } + } + fileContent := fileBytes + var buf bytes.Buffer + compressor, err := zlib.NewWriterLevelDict(&buf, zlib.BestCompression, fileBytes) + if err != nil { + fmt.Println("Compression failed") + } else { + compressor.Write(fileBytes) + compressor.Close() + fileContent = buf.Bytes() + } + fileConStr := baseStdEncode(fileContent) + var cfr models.CvrfSaRecord + cfr.OpenEulerSANum = openEulerSANum + if fixFlag == UNAFFECTFLAG { + cfr.Md5 = fileMd5 + } + //cfr.Md5 = fileMd5 + tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") + if tbErr != nil || cfr.Id == 0 { + cfr.CreateTime = common.GetCurTime() + if fixFlag == UNAFFECTFLAG { + cfr.Md5 = fileMd5 + cfr.CveNum = " " + cfr.PackName = " " + } + cfr.OpenEulerSANum = openEulerSANum + cfr.XmlContent = fileConStr + cfr.Status = 1 + cfr.IsExport = 1 + cfr.AffectFlag = fixFlag + cfr.SortOpenEulerSANum = newText + num, iErr := models.InsertCvrfRecord(&cfr) + if iErr != nil || num == 0 { + logs.Error("InsertCvrfRecord, ", iErr) + } + } else { + cfr.UpdateTime = common.GetCurTime() + if fixFlag == UNAFFECTFLAG { + cfr.Md5 = fileMd5 + } + cfr.XmlContent = fileConStr + cfr.Status = 1 + cfr.IsExport = 1 + cfr.AffectFlag = fixFlag + cfr.SortOpenEulerSANum = newText + uErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", "UpdateMd5", "XmlContent", + "Status", "IsExport", "AffectFlag", "SortOpenEulerSANum") + if uErr != nil { + logs.Error("UpdateCvrfRecord, ", uErr) + } + } + return nil +} + +// File Upload +func PostFile(filename string, targetUrl string) error { + upfileErr := ObsUploadFile(targetUrl, filename) + if upfileErr != nil { + logs.Error("upfileErr: ", upfileErr) + } + return upfileErr +} + +//downlaod file +func GetFile(url, filePath string) error { + res, err := http.Get(url) + if err != nil { + logs.Error("GetFile1, err: ", err) + return err + } + f, err := os.Create(filePath) + if err != nil { + logs.Error("GetFile2, err: ", err) + return err + } + io.Copy(f, res.Body) + return nil +} + +func ReadWriteFile(filePath string, fileSlice []string) error { + if len(fileSlice) == 0 { + return errors.New("file content does not exist") + } + localDataSlice := make([]string, 0) + localFileSlice := common.SliceRemoveDup(fileSlice) + for _, lf := range localFileSlice { + localDataSlice = append(localDataSlice, lf.(string)) + } + sort.Strings(localDataSlice) + fileContent := strings.Join(localDataSlice, "\n") + fisExist, _ := PathExists(filePath) + if fisExist { + os.Remove(filePath) + } + file, err := os.Create(filePath) + if err != nil { + logs.Error("open file err", err) + return err + } + defer file.Close() + _, err = file.Write([]byte(fileContent)) + if err != nil { + logs.Error(err) + } + return nil +} + +func ReadFileAll(filePath string) string { + file, err := os.Open(filePath) + if err != nil { + fmt.Println(err) + return "" + } + defer file.Close() + fileinfo, err := file.Stat() + if err != nil { + fmt.Println(err) + return "" + } + filesize := fileinfo.Size() + buffer := make([]byte, filesize) + + bytesread, err := file.Read(buffer) + if err != nil { + fmt.Println(err) + return "" + } + logs.Info("bytes read: ", bytesread) + logs.Info("bytestream to string: ", string(buffer)) + return string(buffer) +} + +type ReplaceHelper struct { + OldPath string + NewPath string + OldText string + NewText string +} + +func (h *ReplaceHelper) DoWrok() error { + return filepath.Walk(h.OldPath, h.walkCallback) +} + +func (h ReplaceHelper) walkCallback(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f == nil { + return nil + } + if f.IsDir() { + return nil + } + buf, err := ioutil.ReadFile(path) + if err != nil { + return err + } + content := string(buf) + newContent := strings.Replace(content, h.OldText, h.NewText, -1) + ioutil.WriteFile(h.NewPath, []byte(newContent), 0) + return err +} diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go new file mode 100644 index 0000000000000000000000000000000000000000..d809e1b9ef6f1246995fe97091e9eb3bf9bedaa1 --- /dev/null +++ b/cve-vulner-manager/taskhandler/excel.go @@ -0,0 +1,1336 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "github.com/360EntSecGroup-Skylar/excelize/v2" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +const CVRFFKEY = "cvrfFileKey" +const BRANCHSKEY = "opBranchsKey" +const UNAFFECTCVRFKEY = "unaffectcvrfkey" +const FIXEDFLAGE = 1 +const UNAFFECTFLAG = 2 + +//CveExcel Excel export client +type CveExcel struct { + ExcelName string //excel name + ExcelHandel *excelize.File //excel File handle + CveSheetIdx int //CVE sheet index + CveSheetName string // CVE sheet Name + SecNoticeSheetIdx int //Security bulletin index + SecNoticeSheetName string //Security bulletin sheet name + SecNoticePrefix string //Security bulletin prefix + SecNoticeSuffixIdx int64 //Security bulletin suffix start value + InfProductSheetName string //CVE Affected products + InfProductSheetIndex int + PackageURLSheetName string + PackageURLSheetIndex int +} + +type IssueAndPkg struct { + IssueMap map[int64]models.PullRequestIssue + IssuePkg string + Repo string + IssueId int64 +} + +var fillLock sync.Mutex +var wgTrigger sync.WaitGroup + +//GenerateCveExcel Generate Excel documents based on data. +//param snPrefix means security notice prefix. +//param snSuffix means security notice suffix append start value. +//param forceRewrite means whether to force the document to be rewritten. +func GenerateCveExcel(excelName, snPrefix string, snSuffix int64, forceRewrite bool) (err error) { + //Query the data to be exported. + count := models.GetCanExportVulnCenterCount() + if count < 1 { + return errors.New("No data to export! ") + } + //init excelClient + ec := CveExcel{} + err = ec.Init(excelName, snPrefix, snSuffix) + if err != nil { + return err + } + mode := ec.InitFileHandle(forceRewrite) + if mode == 0 { + ec.InitSheet() + err = ec.FillHeader() + if err != nil { + logs.Error(err) + } + } + ec.FillContent(count) + //generate excel + return ec.Save(mode) +} + +//GenerateCveExcelByTrigger Generate cve&security notice excel file by trigger +func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime string, snSuffix int64, + forceRewrite bool, pkgList []models.ExcelPackage, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cvexml *[]CveXml, + dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm) (err error) { + if len(pkgList) == 0 { + return errors.New("No data to export! ") + } + //init excelClient + ec := CveExcel{} + err = ec.Init(excelName, snPrefix, snSuffix) + if err != nil { + logs.Error("excelName: ", excelName, err) + return err + } + mode := ec.InitFileHandle(forceRewrite) + if mode == 0 { + ec.InitSheet() + err = ec.FillHeader() + if err != nil { + logs.Error(err) + } + } + ec.FillContentTrigger(pkgList, startTime, affectBranch, cvrfFileList, + componentMap, cvfrFileMap, cvexml, dpdates, + securityNotice, packRpmx) + return ec.Save(mode) +} + +//Init init excel client +func (ec *CveExcel) Init(excelName, snPrefix string, snSuffix int64) (err error) { + if excelName == "" || !(strings.HasSuffix(excelName, ".xlsx") || strings.HasSuffix(excelName, "xls")) { + err = errors.New("excel name illegal") + return + } + ec.ExcelName = excelName + ec.SecNoticePrefix = snPrefix + ec.SecNoticeSuffixIdx = snSuffix + ec.CveSheetIdx = 0 + ec.CveSheetName = "CVE" + ec.SecNoticeSheetIdx = 1 + ec.SecNoticeSheetName = "安全公告" + ec.InfProductSheetIndex = 2 + ec.InfProductSheetName = "CVE影响的产品" + ec.PackageURLSheetIndex = 3 + ec.PackageURLSheetName = "安全公告升级包链接" + return nil +} + +//InitFileHandle Initialize the file handle. +//param forceRewrite is true it means will create a new file otherwise it means append or new. +//the return value wm is 0 for new creation, and 1 for append. +func (ec *CveExcel) InitFileHandle(forceRewrite bool) (wm int8) { + if forceRewrite { + ec.ExcelHandel = excelize.NewFile() + return 0 + } + s, err := os.Stat(ec.ExcelName) + if err != nil { + ec.ExcelHandel = excelize.NewFile() + return 0 + } + if !s.IsDir() { + file, err := excelize.OpenFile(ec.ExcelName) + if err != nil { + ec.ExcelHandel = excelize.NewFile() + return 0 + } + ec.ExcelHandel = file + return 1 + + } + ec.ExcelHandel = excelize.NewFile() + return 0 + +} + +//InitSheet init excel sheet +func (ec *CveExcel) InitSheet() { + ec.SecNoticeSheetIdx = ec.ExcelHandel.NewSheet(ec.SecNoticeSheetName) + ec.InfProductSheetIndex = ec.ExcelHandel.NewSheet(ec.InfProductSheetName) + ec.PackageURLSheetIndex = ec.ExcelHandel.NewSheet(ec.PackageURLSheetName) + sn := ec.ExcelHandel.GetSheetName(ec.CveSheetIdx) + ec.ExcelHandel.SetSheetName(sn, ec.CveSheetName) +} + +//FillHeader fill the excel sheet header +func (ec *CveExcel) FillHeader() (err error) { + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "A1", "CVE编号") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "A1", "A2") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "B1", "概要") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "B1", "B2") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "C1", "Cvss Score") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "C1", "D1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "c2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "D2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "E1", "Attack Vector") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "E1", "F1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "E2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "F2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "G1", "Attack Complexity") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "G1", "H1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "G2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "H2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "I1", "Privileges Required") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "I1", "J1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "I2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "J2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "K1", "User Interaction") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "K1", "L1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "K2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "L2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "M1", "Scope") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "M1", "N1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "M2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "N2", "OpenEuler") + if err != nil { + return + } + + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "O1", "Confidentiality") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "O1", "P1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "O2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "P2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "Q1", "Integrity") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "Q1", "R1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "Q2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "R2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "S1", "Availability") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "S1", "T1") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "S2", "NVD") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "T2", "OpenEuler") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "U1", "评分类型") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "U1", "U2") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "V1", "安全公告") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "V1", "V2") + if err != nil { + return + } + err = ec.ExcelHandel.SetCellValue(ec.CveSheetName, "W1", "公告时间") + if err != nil { + return + } + err = ec.ExcelHandel.MergeCell(ec.CveSheetName, "W1", "W2") + if err != nil { + return + } + //Security bulletin header generation + sh := []interface{}{"公告编号", "CVE编号", "简介", "概要", "主题", "描述", "影响组件", "影响产品", "影响包名", "参考", "公告时间"} + err = ec.ExcelHandel.SetSheetRow(ec.SecNoticeSheetName, "A1", &sh) + if err != nil { + return err + } + ih := []interface{}{"CVE编号", "影响的产品", "包", "状态"} + err = ec.ExcelHandel.SetSheetRow(ec.InfProductSheetName, "A1", &ih) + if err != nil { + return err + } + ph := []interface{}{"包名", "包下载链接"} + err = ec.ExcelHandel.SetSheetRow(ec.PackageURLSheetName, "A1", &ph) + if err != nil { + return err + } + return nil +} + +//FillContent fill the excel content +func (ec *CveExcel) FillContent(count int64) { + pageSize := 50 + pageCount := count / int64(pageSize) + if count%int64(pageSize) > 0 { + pageCount++ + } + for i := int64(0); i < pageCount; i++ { + off := i * int64(pageSize) + err := ec.handleWriteContent(off, pageSize) + if err != nil { + logs.Error(err) + } + } +} + +func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cvexml *[]CveXml, + dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm) { + pl := len(pkgList) + pageSize := 10 + pc := pl / 10 + if pl%10 > 0 { + pc++ + } + cd := make(chan []IssueAndPkg) + start := 0 + end := 0 + for i := 0; i < pc; i++ { + start = i * pageSize + end = (i + 1) * pageSize + if end > pl { + end = pl + } + wgTrigger.Add(1) + go getDateByGite(pkgList[start:end], startTime, cd, affectBranch) + } + for i := 0; i < pc; i++ { + wgTrigger.Add(1) + go ec.handleGiteData(cd, affectBranch, cvexml, dpdates, + securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) + } + wgTrigger.Wait() +} + +func FileExist(fileList []string) bool { + if len(fileList) > 0 { + for _, filex := range fileList { + _, isErr := os.Stat(filex) + if isErr != nil { + return false + } + } + } + return true +} + +func DelFile(fileList []string) { + if len(fileList) > 0 { + for _, filex := range fileList { + err := os.Remove(filex) + if err != nil { + logs.Error(err) + } + } + } +} + +func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { + list, err := models.GetCanExportCveData(off, size) + if err != nil { + return err + } + var cvrfFileList map[string][]string + var componentMap map[string]ComponentInfo + var cvfrFileMap map[string]CvrfSa + lz := len(list) + if lz > 0 { + for _, v := range list { + if v.Num == 1 { + ec.setContentRow(v, "", cvrfFileList, componentMap, cvfrFileMap) + } else if v.Num > 1 { + //1.Obtain issue_tpl according to cve_num, if all issue_status == 2 0r issue_status == 6, then data can be exported + list, err := models.GetIssueTplByCveNum(v.CveNum) + if err != nil { + logs.Error(err) + continue + } + mergerList := make([]string, 0) + canMerger := true + for _, t := range list { + if t.Status != 3 && t.Status != 4 { + canMerger = false + break + } + if t.IssueStatus == 2 { + mergerList = append(mergerList, strconv.FormatInt(t.CveId, 10)) + } + } + //2.Find the corresponding ExcelExport data with the highest score from the template data of issue_status == 2 + if canMerger && len(mergerList) > 0 { + canExport, err := models.GetCanExportCveDataSameNum(strings.Join(mergerList, ",")) + if err != nil { + logs.Error(err) + } + ep := canExport[0] + if len(canExport) > 1 { + canExport = canExport[1:] + for _, ex := range canExport { + ep.Introduction = ep.Introduction + "\n" + ex.Introduction + ep.Summary = ep.Summary + "\n" + ex.Summary + ep.Theme = ep.Theme + "\n" + ex.Theme + ep.Description = ep.Description + "\n" + ex.Description + ep.OwnedComponent = ep.OwnedComponent + "\n" + ex.OwnedComponent + ep.InfluenceComponent = ep.InfluenceComponent + "\n" + ex.InfluenceComponent + ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct + } + } + ec.setContentRow(ep, "", cvrfFileList, componentMap, cvfrFileMap) + } + } + } + } + return nil +} + +func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, + affectBranch string, cvexml *[]CveXml, dpdates *Updates, + securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) (err error) { + lz := len(list) + if lz > 0 { + for _, v := range list { + if v.OrganizateId == 2 { + logs.Error("opengauss, data: ", v) + continue + } + affectBool := affectBrachRep(&v, affectBranch) + if !affectBool { + logs.Error("Unaffected version, data: ", v) + continue + } + fillLock.Lock() + ec.setContentRow(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap) + fillLock.Unlock() + fillLock.Lock() + BuildXml(cvexml, &v, securityNotice, packRpmx, affectBranch) + fillLock.Unlock() + if v.Num == 1 { + // The current data num==1 will not be processed + //fillLock.Lock() + //BuildXml(cvexml, &v, securityNotice, packRpmx) + //fillLock.Unlock() + logs.Info("Num:1, info: ", v) + } else if v.Num > 1 { + logs.Info("Num: ", v.Num, ", info: ", v) + //Handling CVE_NUM duplicates + //1.Obtain issue_tpl according to cve_num, if all issue_status == 2 0r issue_status == 6, then data can be exported + list, err := models.GetIssueTplByCveNum(v.CveNum) + if err != nil { + logs.Error("GetIssueTplByCveNum, err: ", err) + continue + } + mergerList := make([]string, 0) + canMerger := true + for _, t := range list { + if t.Status != 3 && t.Status != 4 { + canMerger = false + break + } + if t.IssueStatus == 2 { + mergerList = append(mergerList, strconv.FormatInt(t.CveId, 10)) + } + } + if canMerger && len(mergerList) > 0 { + canExport, err := models.GetCanExportCveDataSameNum(strings.Join(mergerList, ",")) + if err != nil { + logs.Error("GetCanExportCveDataSameNum, err: ", err) + } + // Fill in the content of the xml file + canExport1 := make([]models.ExcelExport, len(canExport)) + copy(canExport1, canExport) + //addXmlData(canExport1, cvexml, affectBranch, securityNotice, packRpmx) + // Fill in the content of the excel file + ep := canExport[0] + ep.SecID = v.SecID + //affectBrachRep(&ep, affectBranch) + if ep.AffectProduct != "" && len(ep.AffectProduct) > 1 { + if ep.AffectProduct != affectBranch { + ep.Introduction = strings.ReplaceAll(ep.Introduction, ep.AffectProduct, affectBranch) + ep.Theme = strings.ReplaceAll(ep.Theme, ep.AffectProduct, affectBranch) + ep.AffectProduct = affectBranch + } + } + if len(canExport) > 1 { + canExport = canExport[1:] + m := make(map[string]struct{}) + m[ep.OwnedComponent] = struct{}{} + for _, ex := range canExport { + affectBool := affectBrachRep(&ex, affectBranch) + if !affectBool { + logs.Error("Unaffected version, data: ", ex) + continue + } + //component repeat do not append + if _, ok := m[ex.OwnedComponent]; !ok { + ep.Introduction = ep.Introduction + "\n" + ex.Introduction + ep.Summary = ep.Summary + "\n" + ex.Summary + ep.Theme = ep.Theme + "\n" + ex.Theme + ep.Description = ep.Description + "\n" + ex.Description + ep.InfluenceComponent = ep.InfluenceComponent + "\n" + ex.InfluenceComponent + ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct + ep.OwnedComponent = ep.OwnedComponent + "\n" + ex.OwnedComponent + m[ex.OwnedComponent] = struct{}{} + } + } + } + } + } + } + } + return nil +} + +func affectBrachRep(xmlp *models.ExcelExport, affectBranch string) bool { + affectBool := false + if xmlp.AffectProduct != "" && len(xmlp.AffectProduct) > 1 { + if xmlp.AffectProduct != affectBranch { + affectBranchsxList := strings.Split(xmlp.AffectProduct, "/") + for _, ab := range affectBranchsxList { + if ab == affectBranch { + affectBool = true + xmlp.Introduction = strings.ReplaceAll(xmlp.Introduction, xmlp.AffectProduct, affectBranch) + xmlp.Theme = strings.ReplaceAll(xmlp.Theme, xmlp.AffectProduct, affectBranch) + xmlp.AffectProduct = affectBranch + break + } + } + } else { + affectBool = true + } + } + return affectBool +} + +func FindUnaffectBrach(xmlp *models.ExcelExport, affectBranch, accessToken, owner string) bool { + branchArry, _ := GetBranchesInfo(accessToken, owner, xmlp.OwnedComponent, 1) + affectBool := false + if xmlp.AffectedVersion != "" && len(xmlp.AffectedVersion) > 1 { + affSlice := strings.Split(xmlp.AffectedVersion, ",") + if len(affSlice) > 0 { + for _, aff := range affSlice { + aff = common.BranchVersionRep(aff) + if strings.Contains(aff, affectBranch+":") || strings.Contains(aff, affectBranch+"：") { + branchSlice := []string{} + if strings.Contains(aff, ":") { + branchSlice = strings.Split(aff, ":") + } else if strings.Contains(aff, "：") { + branchSlice = strings.Split(aff, "：") + } + if len(branchSlice) == 2 { + if strings.TrimSpace(branchSlice[0]) == affectBranch && + len(strings.TrimSpace(branchSlice[1])) > 1 && + strings.TrimSpace(branchSlice[1]) != "受影响" { + // Query branch information + for _, arr := range branchArry { + if arr == strings.TrimSpace(branchSlice[0]) { + affectBool = true + break + } + } + } + } + } + } + } + } + return affectBool +} + +func addXmlData(canExport []models.ExcelExport, cvexml *[]CveXml, affectBranch string, + securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { + for _, xmlp := range canExport { + affectBool := affectBrachRep(&xmlp, affectBranch) + if !affectBool { + logs.Error("Unaffected version, data: ", xmlp) + continue + } + fillLock.Lock() + BuildXml(cvexml, &xmlp, securityNotice, packRpmx, affectBranch) + fillLock.Unlock() + } +} + +func UpdateUseOpenEulerSaNum() { + useOpenEulerNum := beego.AppConfig.String("xml::use_openeuler_num") + if len(useOpenEulerNum) > 1 { + useOpenEulerNumSlice := strings.Split(useOpenEulerNum, ",") + for _, use := range useOpenEulerNumSlice { + var sax models.SaNumber + sax.OpenEulerSANum = use + sax.Status = 1 + models.GetSaNumber(&sax, "OpenEulerSANum", "Status") + if sax.SaId > 0 { + sax.Status = 2 + sax.UpdateTime = common.GetCurTime() + models.UpdateSaNumber(&sax, "status") + } + var saFile models.SaFileList + saFile.FileName = "cvrf-" + use + ".xml" + models.GetCvrfFileName(&saFile, "FileName") + if saFile.FileId == 0 { + saFile.Status = 1 + saFile.CreateTime = common.GetCurTime() + models.InsertCvrfFileName(&saFile) + } + } + } +} + +func GetOpenEulerSaNum() string { + var sa models.SaNumber + curYears := strconv.Itoa(time.Now().Year()) + models.GetSaNumberByYears(&sa, curYears, 1, 1) + if sa.SaId > 0 { + sa.Status = 2 + sa.UpdateTime = common.GetCurTime() + models.UpdateSaNumber(&sa, "status") + } + return sa.OpenEulerSANum +} + +func StoreComponentInfo(componentMap map[string]ComponentInfo, + v models.ExcelExport, affectBranch string) bool { + repFlag := false + influenceComponent := "" + fileDir := beego.AppConfig.String("fileDir") + common.CreateDir(fileDir) + if len(v.InfluenceComponent) > 1 { + influenceComponent = v.InfluenceComponent + } else { + influenceComponent = v.OwnedComponent + } + packNameStruct, pOk := componentMap[influenceComponent] + if !pOk || len(packNameStruct.OpenEulerSANum) < 3 { + var coponentInfo ComponentInfo + openEulerSANum := GetOpenEulerSaNum() + if openEulerSANum == "" { + openEulerSANum = v.OpenEulerSANum + } + coponentInfo.OpenEulerSANum = openEulerSANum + coponentInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum+".xml") + coponentInfo.OwnedComponent = influenceComponent + coponentInfo.UpdateFlag = 2 + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) + coponentInfo.OpenEulerScore = openEulerScoreSlice + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + branchCve := make(map[string]BrachCveInfo) + var brachCveInfo BrachCveInfo + openEulerSANum1 := GetOpenEulerSaNum() + if openEulerSANum1 == "" { + openEulerSANum1 = v.OpenEulerSANum + } + brachCveInfo.OpenEulerSANum = openEulerSANum1 + brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") + brachCveInfo.CveNumSlice = cveNumSlice + brachCveInfo.OpenEulerScore = openEulerScoreSlice + branchCve[affectBranch] = brachCveInfo + coponentInfo.CveNumMap = branchCve + coponentInfo.CveNum = cveNumSlice + componentMap[influenceComponent] = coponentInfo + } else { + packNameStruct.UpdateFlag = 2 + repFlagX := false + if len(packNameStruct.CveNum) > 0 { + for _, cveX := range packNameStruct.CveNum { + if cveX == v.CveNum { + repFlagX = true + break + } + } + if !repFlagX { + packNameStruct.CveNum = append(packNameStruct.CveNum, v.CveNum) + packNameStruct.OpenEulerScore = append(packNameStruct.OpenEulerScore, v.OpenEulerScore) + } + } else { + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + packNameStruct.CveNum = cveNumSlice + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) + packNameStruct.OpenEulerScore = openEulerScoreSlice + } + if len(packNameStruct.CveNumMap) > 0 { + if branchCvex, brOk := packNameStruct.CveNumMap[affectBranch]; !brOk { + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) + var brachCveInfo BrachCveInfo + openEulerSANum1 := GetOpenEulerSaNum() + if openEulerSANum1 == "" { + openEulerSANum1 = v.OpenEulerSANum + } + brachCveInfo.OpenEulerSANum = openEulerSANum1 + brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") + brachCveInfo.CveNumSlice = cveNumSlice + brachCveInfo.OpenEulerScore = openEulerScoreSlice + packNameStruct.CveNumMap[affectBranch] = brachCveInfo + } else { + for _, cve := range branchCvex.CveNumSlice { + if cve == v.CveNum { + repFlag = true + break + } + } + if !repFlag { + branchCvex.CveNumSlice = append(branchCvex.CveNumSlice, v.CveNum) + branchCvex.OpenEulerScore = append(branchCvex.OpenEulerScore, v.OpenEulerScore) + packNameStruct.CveNumMap[affectBranch] = branchCvex + } + } + } else { + branchCve := make(map[string]BrachCveInfo) + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) + var brachCveInfo BrachCveInfo + openEulerSANum1 := GetOpenEulerSaNum() + if openEulerSANum1 == "" { + openEulerSANum1 = v.OpenEulerSANum + } + brachCveInfo.OpenEulerSANum = openEulerSANum1 + brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") + brachCveInfo.CveNumSlice = cveNumSlice + brachCveInfo.OpenEulerScore = openEulerScoreSlice + branchCve[affectBranch] = brachCveInfo + packNameStruct.CveNumMap = branchCve + } + componentMap[influenceComponent] = packNameStruct + } + return repFlag +} + +func SaveCvrfName(cvrfFileList map[string][]string, cvrffileName, mapKey string) { + cvrfFileSlice, cvrfOk := cvrfFileList[mapKey] + if cvrfOk { + fileExist := false + for _, cvrfVuale := range cvrfFileSlice { + if cvrfVuale == cvrffileName { + fileExist = true + } + } + if !fileExist { + cvrfFileSlice = append(cvrfFileSlice, cvrffileName) + cvrfFileList[mapKey] = cvrfFileSlice + } + } else { + cvrfNameSlice := make([]string, 0) + cvrfNameSlice = append(cvrfNameSlice, cvrffileName) + cvrfFileList[mapKey] = cvrfNameSlice + } +} + +func BranchCvrfData(v models.ExcelExport, + affectBranch, cvrffileName string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, + cvfrFileMap map[string]CvrfSa, pkg []models.Package, branchFlag int) { + branchCvrfSaStruct, bCvrfSaOk := cvfrFileMap[cvrffileName] + if !bCvrfSaOk { + // Query whether there is data in the database + var cfr models.CvrfSaRecord + var cvrfsa CvrfSa + if branchFlag == 1 { + cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].OpenEulerSANum + } else { + cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].CveNumMap[affectBranch].OpenEulerSANum + } + tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") + if tbErr == nil && cfr.Id > 0 { + // Download data from file server + // Read file content + readErr := ReadCvrfXml(cvrffileName, &cvrfsa) + if readErr != nil { + cvrfsa = CvrfSa{} + } else { + var componentInfo ComponentInfo + componentInfo = componentMap[v.InfluenceComponent] + componentInfo.UpdateFlag = 1 + componentMap[v.InfluenceComponent] = componentInfo + } + } + BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) + cvfrFileMap[cvrffileName] = cvrfsa + } else { + BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) + cvfrFileMap[cvrffileName] = branchCvrfSaStruct + } + SaveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) +} + +func procCvrfData(v models.ExcelExport, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, + cvfrFileMap map[string]CvrfSa, pkg []models.Package) { + totalCvrffileName := componentMap[v.InfluenceComponent].CvrfFileName + branchCvrffileName := componentMap[v.InfluenceComponent].CveNumMap[affectBranch].CvrfFileName + BranchCvrfData(v, affectBranch, branchCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 2) + BranchCvrfData(v, affectBranch, totalCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 1) +} + +func (ec *CveExcel) setContentRow(v models.ExcelExport, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { + pkg, pkgErr := models.GetCvePackageList(v.SecID, affectBranch) + if pkgErr != nil { + logs.Error(pkgErr) + } + if v.PublicDate == "" { + v.PublicDate = time.Now().Format("2006-01-02") + } + pkgStr := getPkgStr(pkg) + repFlag := StoreComponentInfo(componentMap, v, affectBranch) + if !repFlag { + procCvrfData(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, pkg) + sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, + v.Theme, v.Description, v.InfluenceComponent, + v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} + axis, searched := ec.searchValueInSheet(ec.SecNoticeSheetName, v.InfluenceComponent) + if !searched { + fillErr := ec.fillSecurityNoticeSheet(sn) + if fillErr != nil { + logs.Error(fillErr) + } + } else { + //merge openEuler SA notice data + colReg := regexp.MustCompile(`[A-Z]*`) + col := colReg.FindString(axis) + row := strings.Trim(axis, col) + rCN := "B" + row + rRl := "J" + row + vcn, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rCN) + vcn += ";\n" + v.CveNum + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rCN, vcn) + vrl, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rRl) + vrl += "\n" + v.ReferenceLink + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rRl, vrl) + rSAN := fmt.Sprintf("A%s", row) + vSAN, cellError := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rSAN) + if cellError == nil { + v.OpenEulerSANum = vSAN + } + //merger description + rd := "F" + row + vd, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rd) + dSplit := strings.Split(v.Description, "Security Fix(es):") + if len(dSplit) > 1 { + if !strings.Contains(vd, dSplit[0]) { + vd = dSplit[0] + vd + } + vd += dSplit[1] + } + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, vd) + //Get the highest-rated theme + rd = "E" + row + theme, err := models.GetCanExportTheme(vcn, v.OwnedComponent, affectBranch) + if err == nil && len(theme) > 1 { + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, theme) + } + } + cve := []interface{}{v.CveNum, v.CveBrief, v.NVDScore, v.OpenEulerScore, v.NattackVector, v.OattackVector, + v.NattackComplexity, v.OattackComplexity, v.NprivilegeRequired, v.OprivilegeRequired, v.NuserInteraction, + v.OuserInteraction, v.Nscope, v.Oscope, v.Nconfidentiality, v.Oconfidentiality, v.Nintegrity, v.Ointegrity, + v.Navailability, v.Oavailability, v.ScoreType, v.OpenEulerSANum, v.PublicDate} + shErr := ec.fillCveSheetRow(cve) + if shErr != nil { + logs.Error(shErr) + } + ap := []interface{}{v.CveNum, v.AffectProduct, v.InfluenceComponent, v.AffectStatus} + err := ec.fillAffectProductSheet(ap) + if err != nil { + logs.Error(err) + } + for _, v := range pkg { + pk := []interface{}{v.PackName, v.PackUrl} + if _, ok := ec.searchValueInSheet(ec.PackageURLSheetName, v.PackName); !ok { + err := ec.fillPackageSheet(pk) + if err != nil { + logs.Error(err) + } + } + } + } +} + +func getPkgStr(pkg []models.Package) string { + pkgStr := "" + if len(pkg) == 0 { + return pkgStr + } + var ps []string + for _, v := range pkg { + if v.PackName != "" { + ps = append(ps, v.PackName) + } + } + return strings.Join(ps, ";\n") +} + +func (ec *CveExcel) searchValueInSheet(sheetName, value string) (axis string, searched bool) { + if value == "" { + return + } + sheet, err := ec.ExcelHandel.SearchSheet(sheetName, value) + if err != nil { + return + } + if len(sheet) > 0 { + for _, k := range sheet { + cellValue, _ := ec.ExcelHandel.GetCellValue(sheetName, k) + if cellValue == value { + searched = true + axis = k + break + } + } + return + } + return +} + +func (ec *CveExcel) fillCveSheetRow(row []interface{}) (err error) { + rows, err := ec.ExcelHandel.GetRows(ec.CveSheetName) + if err != nil { + return err + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + err = ec.ExcelHandel.SetSheetRow(ec.CveSheetName, axis, &row) + return err +} + +func (ec *CveExcel) fillSecurityNoticeSheet(row []interface{}) (err error) { + rows, err := ec.ExcelHandel.GetRows(ec.SecNoticeSheetName) + if err != nil { + return err + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + err = ec.ExcelHandel.SetSheetRow(ec.SecNoticeSheetName, axis, &row) + return err +} + +func (ec *CveExcel) fillAffectProductSheet(row []interface{}) (err error) { + rows, err := ec.ExcelHandel.GetRows(ec.InfProductSheetName) + if err != nil { + return err + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + err = ec.ExcelHandel.SetSheetRow(ec.InfProductSheetName, axis, &row) + return err +} + +func (ec *CveExcel) fillPackageSheet(row []interface{}) (err error) { + rows, err := ec.ExcelHandel.GetRows(ec.PackageURLSheetName) + if err != nil { + return err + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + err = ec.ExcelHandel.SetSheetRow(ec.PackageURLSheetName, axis, &row) + return err +} + +//Save save the excel content to file +func (ec *CveExcel) Save(md int8) error { + if md == 0 { + return ec.ExcelHandel.SaveAs(ec.ExcelName) + } + return ec.ExcelHandel.Save() + +} + +//ExtractPackageData extract the package data by csv file +func ExtractPackageData(lp string) (pkgList []models.ExcelPackage, err error) { + pkgLock.Lock() + defer pkgLock.Unlock() + if lp == "" || path.Ext(lp) != ".CSV" { + return pkgList, errors.New("the file path is error") + } + file, err := os.Open(lp) + if err != nil { + return pkgList, err + } + defer file.Close() + reader := csv.NewReader(file) + for { + line, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return pkgList, err + } + if len(line) >= 3 && len(line[0]) > 3 { + pkgList = append(pkgList, models.ExcelPackage{PubTime: line[0], Repo: line[1], Packages: line[2]}) + } + } + return +} + +func getDataUnaffect(startTime string) ([]models.IssueTemplate, error) { + issueTemp, err := models.GetUnffectIssueNumber(startTime) + if err != nil || issueTemp == nil { + logs.Info("No unaffected data needs to be processed, startTime: ", startTime) + return issueTemp, err + } + return issueTemp, err +} + +func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []IssueAndPkg, affectBranch string) { + defer wgTrigger.Done() + token := beego.AppConfig.String("gitee::git_token") + //token := "8457c66db66955376519059b97e33dd1" + owner := beego.AppConfig.String("gitee::owner") + // Time difference in different time zones + saTimeStampZone, ok := beego.AppConfig.Int64("excel::sa_timestamp_zone") + if ok != nil { + saTimeStampZone = 3600 * 8 + } + st := util.TimeStrToInt(startTime, "2006-01-02") + chData := make([]IssueAndPkg, 0) + for _, v := range pkgList { + //logs.Info("excel, v===>", v) + rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + saTimeStampZone + // Query the current issues that need to be dealt with + issueTemp, err := models.GetIssueNumber(v.Repo) + if err != nil || issueTemp == nil { + logs.Info("No need to deal with ", v.Repo) + continue + } + for _, isTemp := range issueTemp { + //logs.Info("isTemp===>", isTemp) + prList := getRepoIssueAllPR(affectBranch, token, owner, v.Repo, st, rt, isTemp) + //get pull request related issue + repoIssue := make(map[int64]models.PullRequestIssue, 0) + for _, p := range prList { + //getPRRelatedAllIssue(token, owner, v.Repo, st, rt, p.Number, repoIssue) + repoIssue[p.Id] = p + } + if len(repoIssue) > 0 { + chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, + Repo: v.Repo, IssueId: isTemp.IssueId}) + } + } + } + c <- chData +} + +func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, cvexml *[]CveXml, + dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { + defer wgTrigger.Done() + data := <-c + var pkgList []string + for _, v := range data { + //parse package string to list + pkgList = strings.Split(v.IssuePkg, " ") + if len(pkgList) == 0 { + logs.Error("Data is filtered, v.IssuePkg: ", v.IssuePkg) + continue + } + affectIssueProc(v, affectBranch, cvexml, dpdates, securityNotice, packRpmx, + cvrfFileList, componentMap, cvfrFileMap, pkgList, ec) + } +} + +func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, startTime, accessToken, owner string, + unaffectcvrf *UnaffectCvrfSa, unaffectYear int) { + data, err := getDataUnaffect(startTime) + if len(data) > 0 { + for _, v := range data { + issueExist, _ := GetCveSecurityNotice(v.CveNum, v.Repo) + if issueExist { + var center models.VulnCenter + center.CveId = v.CveId + centErr := models.GetVulnCenterByCid(¢er, "cve_id") + if centErr == nil { + center.IsExport = 1 + models.UpdateVulnCenter(¢er, "is_export") + } + continue + } + //save data to excel + el, err := models.GetCanExportExcelData(v.CveNum, v.IssueNum, v.OwnedComponent, v.IssueId) + if err != nil { + logs.Error("GetCanExportExcelData, err: ", err, "tpl.CveNum, tpl.IssueNum: ", v.CveNum, v.IssueNum, v.OwnedComponent) + continue + } + for _, vx := range el { + affectBool := FindUnaffectBrach(&vx, affectBranch, accessToken, owner) + if affectBool { + logs.Info("Unaffected version, data: ", vx.CveNum, vx.OwnedComponent, vx.AffectProduct) + cveNumList := strings.Split(vx.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if cveYears <= unaffectYear { + logs.Error("The unaffected version year is no longer "+ + "processed in the scope, cveYears: ", cveYears, ",unaffectYear: ", unaffectYear) + continue + } + } + } + // Generate unaffected cvrf files + fillLock.Lock() + BuildUnaffectVulnerabilitySet(unaffectcvrf, vx, affectBranch, componentMap) + fillLock.Unlock() + } + } + } + } else { + logs.Error(err) + } +} + +func affectIssueProc(v IssueAndPkg, affectBranch string, cvexml *[]CveXml, + dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, + pkgList []string, ec *CveExcel) { + for _, iv := range v.IssueMap { + tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId} + err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id") + if err != nil { + logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, iv.Repo, v.IssueId) + continue + } + //save data to excel + el, err := models.GetCanExportExcelData(tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent, tpl.IssueId) + if err != nil { + logs.Error("GetCanExportExcelData, err: ", err, "tpl.CveNum, tpl.IssueNum: ", + tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent) + continue + } + // Check whether the cve data has been released sa + issueExist, _ := GetCveSecurityNotice(tpl.CveNum, tpl.Repo) + if issueExist { + var center models.VulnCenter + center.CveId = tpl.CveId + centErr := models.GetVulnCenterByCid(¢er, "cve_id") + if centErr == nil { + center.IsExport = 1 + models.UpdateVulnCenter(¢er, "is_export") + } + continue + } + err = models.ReplacePackageByCveId(pkgList, tpl.CveId, affectBranch) + if err != nil { + logs.Error("ReplacePackageByCveId, err: ", err, "tpl.CveId: ", tpl.CveId) + continue + } + err = ec.handleWriteContentSync(el, affectBranch, cvexml, dpdates, + securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) + if err != nil { + logs.Error("handleWriteContentSync, err: ", err) + } + } +} + +func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, + releaseTime int64, isTemp models.IssueTemplate) (prList []models.PullRequestIssue) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests", owner, isTemp.IssueNum) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error("NewRequest, url: ", url, ",err: ", err) + return + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("repo", repo) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error("DefaultClient, url: ", url, ",err: ", err) + return + } + if resp.StatusCode == http.StatusOK { + issuePr := make([]map[string]interface{}, 0) + read, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, url: ", url, ",err: ", err) + return + } + resp.Body.Close() + err = json.Unmarshal(read, &issuePr) + if err != nil { + logs.Error("Unmarshal, url: ", url, ",err: ", err) + return + } + for _, v := range issuePr { + if _, ok := v["id"]; !ok { + continue + } + pr := models.PullRequestIssue{} + if v["state"].(string) == "merged" && v["mergeable"].(bool) { + //mt := v["closed_at"].(string).(time.Time).Local().Unix() + closedAt := v["closed_at"].(string) + mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") + mergedAt := v["merged_at"].(string) + ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") + //logs.Info("======>>mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo, ", pr: ", v) + //ct := v["merged_at"].(string).(time.Time).Local().Unix() + var pt int64 + if mt > 0 && ct > 0 { + if mt > ct { + pt = ct + } else { + pt = mt + } + if pt >= startTime && pt <= releaseTime { + if v["base"].(map[string]interface{})["label"].(string) == affectBranch { + //if v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo { + // if v["base"].(map[string]interface{})["repo"].(map[string]interface{})["namespace"].(map[string]interface{})["path"] == owner { + // pr.Id = int64(v["id"].(float64)) + // pr.Number = isTemp.IssueNum + // pr.CveNumber = isTemp.CveNum + // pr.Repo = repo + // prList = append(prList, pr) + // } + //} + pr.Id = int64(v["id"].(float64)) + pr.Number = isTemp.IssueNum + pr.CveNumber = isTemp.CveNum + pr.Repo = repo + pr.BrFlag = true + pr.Branch = affectBranch + prList = append(prList, pr) + } + } + } + } + } + } else { + resp.Body.Close() + } + return +} diff --git a/cve-vulner-manager/taskhandler/gauss.go b/cve-vulner-manager/taskhandler/gauss.go new file mode 100644 index 0000000000000000000000000000000000000000..e7fd3067768d5935aaa6a54f931e2c3ef444dbf5 --- /dev/null +++ b/cve-vulner-manager/taskhandler/gauss.go @@ -0,0 +1,1042 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/json" + "errors" + "fmt" + "github.com/360EntSecGroup-Skylar/excelize/v2" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +var localLock sync.Mutex + +type GitInfo struct { + StartTime string + Token string + Owner string +} + +type CveBranch struct { + CveNums []string + BranchMap map[string][]string +} + +type CveComponent struct { + PackMap map[string]CveBranch +} + +// Handling openGauss data types +func GaussUnaffectIssue(branchVersion []string, gi GitInfo) { + for _, bv := range branchVersion { + data, gdErr := models.QueryGaussIssueData(gi.StartTime) + if len(data) > 0 { + for _, v := range data { + ogc := models.OpenGaussCveList{CveNum: v.CveNum, PackName: v.OwnedComponent, Status: 3} + ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName", "Status") + if ogc.Id > 0 { + var center models.VulnCenter + center.CveId = v.CveId + center.OrganizationID = 2 + centErr := models.GetVulnCenterByCid(¢er, "CveId", "OrganizationID") + if centErr == nil { + center.IsExport = 1 + models.UpdateVulnCenter(¢er, "is_export") + } + continue + } else { + logs.Info("Continue to execute data, ogcErr: ", ogcErr) + } + // Get specific business data + el, gsErr := models.QueryGaussExportSaData(v.CveNum, v.IssueNum, v.Repo, v.IssueId) + if gsErr != nil { + logs.Error("QueryGaussExportSaData, err: ", gsErr, "tpl.CveNum, tpl.IssueNum: ", + v.CveNum, v.IssueNum, v.OwnedComponent) + return + } + for _, vx := range el { + affectBool := GaussFindUnaffVersion(&vx, bv, gi) + if affectBool { + logs.Info("Unaffected version, data: ", vx.CveNum, vx.OwnedComponent, vx.AffectProduct) + // Generate unaffected cvrf files + localLock.Lock() + UnaffectCveStore(&vx, bv) + localLock.Unlock() + } + } + } + } else { + logs.Error(gdErr) + } + } +} + +// Does not affect the storage and release of cve data +func UnaffectCveStore(xmlp *models.ExcelExport, versions string) { + ogc := models.OpenGaussCveList{CveNum: xmlp.CveNum, PackName: xmlp.OwnedComponent} + ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName") + if ogc.Id == 0 { + logs.Info("ogcErr: ", ogcErr, xmlp.CveNum, xmlp.OwnedComponent) + CreateReleaseCve(&ogc, xmlp, 1, 0) + ogcId, gocErr := models.InsertReleaseCve(&ogc) + if ogcId > 0 { + ogv := models.OpenGaussVersion{CveId: ogcId, CveNum: xmlp.CveNum, RepoVersion: versions} + verErr := models.QueryGaussVersion(&ogv, "CveId", "CveNum", "RepoVersion") + if ogv.VersionId == 0 { + logs.Info("QueryGaussVersion, verErr: ", verErr) + CreateGaussVersion(&ogv, 1, 2, 2, xmlp.CveNum, versions, + "Unaffected", ogcId) + versionId, vErr := models.InsertGaussVersion(&ogv) + if versionId > 0 { + logs.Info("InsertGaussVersion, Data inserted successfully, id: ", versionId) + } else { + logs.Error("InsertGaussVersion, Data insertion failed, err: ", vErr, ogv) + } + } else { + gv := CreateGaussVersion(&ogv, 2, 2, 2, xmlp.CveNum, versions, + "Unaffected", ogcId) + uvErr := models.UpdateGaussVersion(&ogv, gv...) + logs.Info("UpdateGaussVersion, uvErr: ", uvErr) + } + } else { + logs.Error("InsertReleaseCve, gocErr: ", gocErr) + return + } + } else { + updateSlice := CreateReleaseCve(&ogc, xmlp, 2, 0) + upErr := models.UpdateReleaseCve(&ogc, updateSlice...) + if upErr == nil { + ogv := models.OpenGaussVersion{CveId: ogc.Id, CveNum: xmlp.CveNum, RepoVersion: versions} + verErr := models.QueryGaussVersion(&ogv, "CveId", "CveNum", "RepoVersion") + if ogv.VersionId == 0 { + logs.Info("QueryGaussVersion2, verErr: ", verErr) + CreateGaussVersion(&ogv, 1, 2, 2, xmlp.CveNum, versions, + "Unaffected", ogc.Id) + versionId, vErr := models.InsertGaussVersion(&ogv) + if versionId > 0 { + logs.Info("InsertGaussVersion2, Data inserted successfully, id: ", versionId) + } else { + logs.Error("InsertGaussVersion2, Data insertion failed, err: ", vErr, ogv) + } + } else { + gv := CreateGaussVersion(&ogv, 2, 2, 2, xmlp.CveNum, versions, + "Unaffected", ogc.Id) + uvErr := models.UpdateGaussVersion(&ogv, gv...) + logs.Info("UpdateGaussVersion2, uvErr: ", uvErr) + } + } + } +} + +func CreateGaussVersion(ogv *models.OpenGaussVersion, flag, affectStatus, ReleaseFlag int8, + cveNum, versions, affectStatusName string, cveId int64) []string { + gv := make([]string, 0) + ogv.CveId = cveId + gv = append(gv, "CveId") + ogv.CveNum = cveNum + gv = append(gv, "CveNum") + ogv.RepoVersion = versions + gv = append(gv, "RepoVersion") + ogv.AffectStatusName = affectStatusName + gv = append(gv, "AffectStatusName") + ogv.AffectStatus = affectStatus + gv = append(gv, "AffectStatus") + ogv.ReleaseFlag = ReleaseFlag + gv = append(gv, "ReleaseFlag") + if flag == 2 { + ogv.UpdateTime = common.GetCurTime() + gv = append(gv, "UpdateTime") + } else { + ogv.CreateTime = common.GetCurTime() + } + return gv +} + +func CreateReleaseCve(ogc *models.OpenGaussCveList, xmlp *models.ExcelExport, flag int, gaussId int64) []string { + updateSlice := make([]string, 0) + cveNumSlice := make([]string, 0) + if len(xmlp.CveNum) > 1 { + cveNumSlice = strings.Split(xmlp.CveNum, "-") + } + years := time.Now().Year() + if len(cveNumSlice) > 2 { + years, _ = strconv.Atoi(cveNumSlice[1]) + } + ogc.GaussId = gaussId + ogc.CveNum = xmlp.CveNum + updateSlice = append(updateSlice, "CveNum") + ogc.Description = xmlp.Description + updateSlice = append(updateSlice, "Description") + ogc.PackName = xmlp.OwnedComponent + updateSlice = append(updateSlice, "PackName") + ogc.RepoName = xmlp.Repo + updateSlice = append(updateSlice, "RepoName") + ogc.NVDScore = xmlp.NVDScore + updateSlice = append(updateSlice, "NVDScore") + ogc.OpenEulerScore = xmlp.OpenEulerScore + updateSlice = append(updateSlice, "OpenEulerScore") + ogc.NvectorVule = xmlp.NvectorVule + updateSlice = append(updateSlice, "NvectorVule") + ogc.OvectorVule = xmlp.OvectorVule + updateSlice = append(updateSlice, "OvectorVule") + if xmlp.NattackVector != "" { + ogc.NattackVector = common.StrFirstToUpper(strings.ToLower(xmlp.NattackVector)) + } + updateSlice = append(updateSlice, "NattackVector") + if xmlp.NaccessVector != "" { + ogc.NaccessVector = common.StrFirstToUpper(strings.ToLower(xmlp.NaccessVector)) + } + updateSlice = append(updateSlice, "NaccessVector") + ogc.OattackVector = xmlp.OattackVector + updateSlice = append(updateSlice, "OattackVector") + ogc.OaccessVector = xmlp.OaccessVector + updateSlice = append(updateSlice, "OaccessVector") + if xmlp.NattackComplexity != "" { + ogc.NattackComplexity = common.StrFirstToUpper(strings.ToLower(xmlp.NattackComplexity)) + } + updateSlice = append(updateSlice, "NattackComplexity") + if xmlp.NaccessComplexity != "" { + ogc.NaccessComplexity = common.StrFirstToUpper(strings.ToLower(xmlp.NaccessComplexity)) + } + updateSlice = append(updateSlice, "NaccessComplexity") + ogc.OattackComplexity = xmlp.OattackComplexity + updateSlice = append(updateSlice, "OattackComplexity") + ogc.OaccessComplexity = xmlp.OaccessComplexity + updateSlice = append(updateSlice, "OaccessComplexity") + if xmlp.NprivilegeRequired != "" { + ogc.NprivilegeRequired = common.StrFirstToUpper(strings.ToLower(xmlp.NprivilegeRequired)) + } + updateSlice = append(updateSlice, "NprivilegeRequired") + ogc.OprivilegeRequired = xmlp.OprivilegeRequired + updateSlice = append(updateSlice, "OprivilegeRequired") + if xmlp.NuserInteraction != "" { + ogc.NuserInteraction = common.StrFirstToUpper(strings.ToLower(xmlp.NuserInteraction)) + } + updateSlice = append(updateSlice, "NuserInteraction") + ogc.OuserInteraction = xmlp.OuserInteraction + updateSlice = append(updateSlice, "OuserInteraction") + if xmlp.Nscope != "" { + ogc.Nscope = common.StrFirstToUpper(strings.ToLower(xmlp.Nscope)) + } + updateSlice = append(updateSlice, "Nscope") + ogc.Oscope = xmlp.Oscope + updateSlice = append(updateSlice, "Oscope") + if xmlp.Nconfidentiality != "" { + ogc.Nconfidentiality = common.StrFirstToUpper(strings.ToLower(xmlp.Nconfidentiality)) + } + updateSlice = append(updateSlice, "Nconfidentiality") + ogc.Oconfidentiality = xmlp.Oconfidentiality + updateSlice = append(updateSlice, "Oconfidentiality") + if xmlp.Nintegrity != "" { + ogc.Nintegrity = common.StrFirstToUpper(strings.ToLower(xmlp.Nintegrity)) + } + updateSlice = append(updateSlice, "Nintegrity") + ogc.Ointegrity = xmlp.Ointegrity + updateSlice = append(updateSlice, "Ointegrity") + if xmlp.Navailability != "" { + ogc.Navailability = common.StrFirstToUpper(strings.ToLower(xmlp.Navailability)) + } + updateSlice = append(updateSlice, "Navailability") + ogc.Oavailability = xmlp.Oavailability + updateSlice = append(updateSlice, "Oavailability") + if xmlp.Nauthentication != "" { + ogc.Nauthentication = common.StrFirstToUpper(strings.ToLower(xmlp.Nauthentication)) + } + updateSlice = append(updateSlice, "Nauthentication") + ogc.Oauthentication = xmlp.Oauthentication + updateSlice = append(updateSlice, "Oauthentication") + ogc.ScoreType = xmlp.ScoreType + updateSlice = append(updateSlice, "ScoreType") + ogc.FixLabel = xmlp.IssueLabel + updateSlice = append(updateSlice, "FixLabel") + ogc.Status = 1 + updateSlice = append(updateSlice, "Status") + ogc.Years = years + updateSlice = append(updateSlice, "Years") + if flag == 2 { + ogc.UpdateTime = common.GetCurTime() + updateSlice = append(updateSlice, "UpdateTime") + } else { + ogc.CreateTime = common.GetCurTime() + } + return updateSlice +} + +func GaussFindUnaffVersion(xmlp *models.ExcelExport, versions string, gi GitInfo) bool { + branchArry, _ := GetBranchesInfo(gi.Token, gi.Owner, xmlp.Repo, 2) + if len(branchArry) == 0 { + gaussVersion := beego.AppConfig.String("opengauss::gauss_version") + if len(gaussVersion) > 0 { + gaussVerSlice := strings.Split(gaussVersion, ",") + if len(gaussVerSlice) > 0 { + branchArry = append(branchArry, gaussVerSlice...) + } + } + } + affectBool := false + if xmlp.AffectedVersion != "" && len(xmlp.AffectedVersion) > 1 { + affSlice := strings.Split(xmlp.AffectedVersion, ",") + if len(affSlice) > 0 { + for _, aff := range affSlice { + if strings.Contains(aff, versions+":") || strings.Contains(aff, versions+"：") { + branchSlice := make([]string, 0) + if strings.Contains(aff, ":") { + branchSlice = strings.Split(aff, ":") + } else if strings.Contains(aff, "：") { + branchSlice = strings.Split(aff, "：") + } + if len(branchSlice) == 2 { + if strings.TrimSpace(branchSlice[0]) == versions && + len(strings.TrimSpace(branchSlice[1])) > 1 && + strings.TrimSpace(branchSlice[1]) != "受影响" { + // Query branch information + for _, arr := range branchArry { + if arr == strings.TrimSpace(branchSlice[0]) { + affectBool = true + break + } + } + } + } + } + } + } + } + return affectBool +} + +// Download the excel file that needs to be processed and parse the data +func DownloadFileAndParse(fExcel models.OpenGaussDownloadFile, startTime, + gaussDir string, cveComponent *CveComponent) { + fileSlice := make([]string, 0) + if len(fExcel.FilePath) > 1 { + fileSlice = strings.Split(fExcel.FilePath, "/") + } + localPath := filepath.Join(gaussDir, fileSlice[len(fileSlice)-1]) + // Download files from the Internet + dErr := downloadGaussPackageFile(localPath, fExcel.FilePath) + if dErr != nil { + logs.Error(dErr) + return + } + pkgList, err := ReadExcelFile(localPath, fExcel.RepoVersion) + if err != nil { + logs.Error(err) + return + } else { + pErr := ParseExcelFile(pkgList, fExcel, startTime, cveComponent) + if pErr != nil { + logs.Error(pErr) + } + } +} + +func ParseExcelFile(pkgList []models.GaussExcelTag, fExcel models.OpenGaussDownloadFile, + startTime string, cveComponent *CveComponent) error { + pl := len(pkgList) + pageSize := 10 + pc := pl / 10 + if pl%10 > 0 { + pc++ + } + cd := make(chan []IssueAndPkg) + start := 0 + end := 0 + for i := 0; i < pc; i++ { + start = i * pageSize + end = (i + 1) * pageSize + if end > pl { + end = pl + } + wgTrigger.Add(1) + go GuassGetDateByGite(pkgList[start:end], cd, startTime, fExcel.RepoVersion) + } + for i := 0; i < pc; i++ { + wgTrigger.Add(1) + go GaussHandleGiteData(cd, fExcel, cveComponent) + } + wgTrigger.Wait() + return nil +} + +func GaussHandleGiteData(c <-chan []IssueAndPkg, fExcel models.OpenGaussDownloadFile, cveComponent *CveComponent) { + defer wgTrigger.Done() + data := <-c + var pkgList []string + for _, v := range data { + pkgList = strings.Split(v.IssuePkg, " ") + if len(pkgList) == 0 { + logs.Error("Data is filtered, v.IssuePkg: ", v.IssuePkg) + continue + } + GaussAffectIssue(v, pkgList, fExcel, cveComponent) + } +} + +func GaussAffectIssue(v IssueAndPkg, pkgList []string, fExcel models.OpenGaussDownloadFile, cveComponent *CveComponent) { + for _, iv := range v.IssueMap { + tpl := models.IssueTemplate{IssueNum: iv.Number, IssueId: v.IssueId} + err := models.GetIssueTemplateByColName(&tpl, "issue_num", "issue_id") + if err != nil { + logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, v.Repo, v.IssueId) + continue + } + //save data to db + el, err := models.QueryGaussExportSaData(tpl.CveNum, tpl.IssueNum, tpl.Repo, tpl.IssueId) + if err != nil { + logs.Error("QueryGaussExportSaData, err: ", err, "tpl.CveNum, tpl.IssueNum: ", + tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent) + continue + } + err = GaussHandleContentToDbSync(el, pkgList, fExcel, cveComponent) + if err != nil { + logs.Error("GaussHandleContentToDbSync, err: ", err) + } + } +} + +func GaussHandleContentToDbSync(list []models.ExcelExport, pkgList []string, + fExcel models.OpenGaussDownloadFile, cveComponent *CveComponent) (err error) { + lz := len(list) + if lz > 0 { + for _, v := range list { + if v.OrganizateId != 2 { + logs.Error("openEuler, data: ", v) + continue + } + //affectProduct := v.AffectProduct + affectBool := affectBrachRep(&v, fExcel.RepoVersion) + if !affectBool { + logs.Error("Unaffected version, data: ", v) + continue + } + localLock.Lock() + AddSaToDb(v, fExcel.RepoVersion, pkgList, cveComponent) + localLock.Unlock() + } + } + return nil +} + +func AddSaToDb(v models.ExcelExport, affectBranch string, pkgList []string, cveComponent *CveComponent) { + if v.PublicDate == "" { + v.PublicDate = time.Now().Format("2006-01-02") + } + gaussTempId := int64(0) + ogl := models.OpenGaussListTemp{InfluenceComponent: v.OwnedComponent, AffectProduct: affectBranch} + oErr := models.QuerySaDataByPackName(&ogl) + if ogl.GaussTempId > 0 { + gaussTempId = ogl.GaussTempId + oglSlice := CreateSaData(v, &ogl, affectBranch) + uErr := models.UpdateSaData(&ogl, oglSlice...) + if uErr != nil { + logs.Error("UpdateSaData, uErr: ", uErr) + } + } else { + logs.Info("QuerySaDataByPackName, err: ", oErr) + CreateSaData(v, &ogl, affectBranch) + id, iErr := models.InsertSaData(&ogl) + if id > 0 { + gaussTempId = id + prErr := ParseRpmPackage(pkgList, gaussTempId, v.OwnedComponent, v.PublicDate, affectBranch) + if prErr != nil { + logs.Error("ParseRpmPackage, prErr: ", prErr) + } + } else { + logs.Error("InsertSaData, iErr: ", iErr) + } + } + AffectCveStore(&v, affectBranch, gaussTempId) + // Global data store + GlobalStoreData(v, affectBranch, cveComponent) +} + +// store cve info +func GlobalStoreData(v models.ExcelExport, affectBranch string, cveComponent *CveComponent) { + if cveComponent.PackMap != nil && len(cveComponent.PackMap) > 0 { + cb, ok := cveComponent.PackMap[v.OwnedComponent] + if ok { + if len(cb.CveNums) > 0 { + cveFlag := false + for _, cn := range cb.CveNums { + if cn == v.CveNum { + cveFlag = true + break + } + } + if !cveFlag { + cb.CveNums = append(cb.CveNums, v.CveNum) + } + } else { + cb.CveNums = make([]string, 0) + cb.CveNums = append(cb.CveNums, v.CveNum) + } + if len(cb.BranchMap) > 0 { + af, afOk := cb.BranchMap[affectBranch] + if afOk { + if len(af) > 0 { + cfFlag := false + for _, ch := range af { + if ch == v.CveNum { + cfFlag = true + break + } + } + if !cfFlag { + af = append(af, v.CveNum) + cb.BranchMap[affectBranch] = af + } + } else { + af = make([]string, 0) + af = append(af, v.CveNum) + cb.BranchMap[affectBranch] = af + } + } else { + af = make([]string, 0) + af = append(af, v.CveNum) + cb.BranchMap[affectBranch] = af + } + } else { + cb.BranchMap = make(map[string][]string) + af := make([]string, 0) + af = append(af, v.CveNum) + cb.BranchMap[affectBranch] = af + } + cveComponent.PackMap[v.OwnedComponent] = cb + } else { + cveNums := make([]string, 0) + cveNums = append(cveNums, v.CveNum) + branchMap := make(map[string][]string) + branchMap[affectBranch] = cveNums + var cb CveBranch + cveNums1 := make([]string, 0) + cveNums1 = append(cveNums1, v.CveNum) + cb.CveNums = cveNums1 + cb.BranchMap = branchMap + cveComponent.PackMap[v.OwnedComponent] = cb + } + } else { + cveNums := make([]string, 0) + cveNums = append(cveNums, v.CveNum) + branchMap := make(map[string][]string) + branchMap[affectBranch] = cveNums + var cb CveBranch + cveNums1 := make([]string, 0) + cveNums1 = append(cveNums1, v.CveNum) + cb.CveNums = cveNums1 + cb.BranchMap = branchMap + packMap := make(map[string]CveBranch) + packMap[v.OwnedComponent] = cb + cveComponent.PackMap = packMap + } +} + +func AddCveAffectProduct(cveNum, versions string, cveId int64) { + models.DeleteGaussVersionByIdStatus(cveId) + ogv := models.OpenGaussVersion{CveId: cveId, CveNum: cveNum, RepoVersion: versions} + CreateGaussVersion(&ogv, 1, 1, 1, cveNum, versions, "Fixed", cveId) + versionId, vErr := models.InsertGaussVersion(&ogv) + if versionId > 0 { + logs.Info("AffectCveStore, InsertGaussVersion, Data inserted successfully, id: ", versionId) + } else { + logs.Error("AffectCveStore, InsertGaussVersion, Data insertion failed, err: ", vErr, ogv) + } +} + +// Does affect the storage and release of cve data +func AffectCveStore(xmlp *models.ExcelExport, versions string, gaussTempId int64) { + ogc := models.OpenGaussCveList{CveNum: xmlp.CveNum, PackName: xmlp.OwnedComponent} + ogcErr := models.QueryReleaseCve(&ogc) + if ogc.Id == 0 { + logs.Error(ogcErr) + CreateReleaseCve(&ogc, xmlp, 1, gaussTempId) + ogcId, gocErr := models.InsertReleaseCve(&ogc) + if ogcId > 0 { + AddCveAffectProduct(xmlp.CveNum, versions, ogcId) + } else { + logs.Error("AffectCveStore, InsertReleaseCve, gocErr: ", gocErr) + return + } + } else { + updateSlice := CreateReleaseCve(&ogc, xmlp, 2, gaussTempId) + upErr := models.UpdateReleaseCve(&ogc, updateSlice...) + if upErr == nil { + AddCveAffectProduct(xmlp.CveNum, versions, ogc.Id) + } + } +} + +func CreateSaData(v models.ExcelExport, ogl *models.OpenGaussListTemp, affectBranch string) []string { + oglSlice := make([]string, 0) + if !strings.Contains(ogl.CveNums, v.CveNum) { + ogl.CveNums = ogl.CveNums + ";" + v.CveNum + oglSlice = append(oglSlice, "CveNums") + } else { + return oglSlice + } + ogl.AffectProduct = affectBranch + ogl.InfluenceComponent = v.OwnedComponent + ogl.PublicDate = v.PublicDate + oglSlice = append(oglSlice, "PublicDate") + ogl.Status = 1 + oglSlice = append(oglSlice, "Status") + if ogl.GaussTempId > 0 { + ogl.UpdateTime = common.GetCurTime() + oglSlice = append(oglSlice, "UpdateTime") + dSplit := strings.Split(v.Description, "Security Fix(es):") + if len(dSplit) > 1 { + if !strings.Contains(ogl.Description, dSplit[0]) { + ogl.Description = dSplit[0] + ogl.Description + } + ogl.Description += dSplit[1] + } + oglSlice = append(oglSlice, "Description") + if ogl.CveLevelValue > 0 { + switch strings.ToLower(v.CveLevel) { + case "critical": + if ogl.CveLevelValue > 1 { + ogl.CveLevelValue = 1 + ogl.CveLevel = v.CveLevel + } + case "high": + if ogl.CveLevelValue > 2 { + ogl.CveLevelValue = 2 + ogl.CveLevel = v.CveLevel + } + case "medium": + if ogl.CveLevelValue > 3 { + ogl.CveLevelValue = 3 + ogl.CveLevel = v.CveLevel + } + default: + ogl.CveLevelValue = 4 + ogl.CveLevel = v.CveLevel + } + } else { + switch strings.ToLower(v.CveLevel) { + case "critical": + ogl.CveLevelValue = 1 + case "high": + ogl.CveLevelValue = 2 + case "medium": + ogl.CveLevelValue = 3 + default: + ogl.CveLevelValue = 4 + } + ogl.CveLevel = v.CveLevel + } + oglSlice = append(oglSlice, "CveLevelValue") + oglSlice = append(oglSlice, "CveLevel") + theme, err := models.GetGaussExportThem(ogl.CveNums, v.OwnedComponent, affectBranch) + if err == nil && len(theme) > 1 { + ogl.Theme = theme + } + oglSlice = append(oglSlice, "Theme") + } else { + ogl.CreateTime = common.GetCurTime() + ogl.Description = v.Description + ogl.CveLevel = v.CveLevel + switch strings.ToLower(v.CveLevel) { + case "critical": + ogl.CveLevelValue = 1 + case "high": + ogl.CveLevelValue = 2 + case "medium": + ogl.CveLevelValue = 3 + default: + ogl.CveLevelValue = 4 + } + ogl.Summary = v.Summary + oglSlice = append(oglSlice, "Summary") + ogl.Introduction = v.Introduction + oglSlice = append(oglSlice, "Introduction") + ogl.Theme = v.Theme + ogl.CveNums = v.CveNum + } + ogl.Theme = strings.ReplaceAll(ogl.Theme, "openEuler", "openGauss") + return oglSlice +} + +func ParseRpmPackage(pkgList []string, gaussTempId int64, packageName, publicDate, affectBranch string) error { + if len(pkgList) > 0 { + for _, pg := range pkgList { + pgSlice := strings.Split(pg, ".") + if len(pgSlice) < 3 { + logs.Error("ParseRpmPackage, rpm package structure error, pg: ", pg) + continue + } + groupId := int64(0) + ogg := models.OpenGaussGroup{GroupName: pgSlice[len(pgSlice)-2]} + gErr := models.QueryOpenGaussGroup(&ogg, "GroupName") + if ogg.GroupId == 0 { + ogg.GroupName = pgSlice[len(pgSlice)-2] + ogg.CreateTime = common.GetCurTime() + ogg.Status = 1 + id, iErr := models.InsertOpenGaussGroup(&ogg) + if iErr != nil { + logs.Error("InsertOpenGaussGroup, err:", iErr) + continue + } + groupId = id + } else { + groupId = ogg.GroupId + logs.Info(gErr) + } + platId := int64(0) + ogp := models.OpenGaussPlatform{PlatName: pgSlice[len(pgSlice)-1]} + pErr := models.QueryOpenGaussPlatform(&ogp, "PlatName") + if ogp.PlatId == 0 { + ogp.PlatName = pgSlice[len(pgSlice)-1] + ogp.Status = 1 + ogp.CreateTime = common.GetCurTime() + id, iErr := models.InsertOpenGaussPlatform(&ogp) + if iErr != nil { + logs.Error("InsertOpenGaussPlatform, err:", iErr) + continue + } + platId = id + } else { + platId = ogp.PlatId + logs.Info(pErr) + } + rpmName := strings.Join(pgSlice[:len(pgSlice)-2], ".") + ogk := models.OpenGaussPackage{GroupId: groupId, PlatId: platId, + GaussId: gaussTempId, PackageName: packageName, RpmName: rpmName, RepoVersion: affectBranch} + opErr := models.QueryOpenGaussPackage(&ogk, "GroupId", "PlatId", "GaussId", "PackageName", "RpmName", "RepoVersion") + if ogk.PackId > 0 { + ogk.Status = 1 + ogk.ReleaseDate = publicDate + ogk.UpdateTime = common.GetCurTime() + upErr := models.UpdateOpenGaussPackage(&ogk, "Status", "ReleaseDate", "UpdateTime") + if upErr != nil { + logs.Error("UpdateOpenGaussPackage, upErr: ", upErr) + } + } else { + logs.Info("QueryOpenGaussPackage, opErr: ", opErr) + ogk = models.OpenGaussPackage{GroupId: groupId, PlatId: platId, GaussId: gaussTempId, + PackageName: packageName, RpmName: rpmName, CreateTime: common.GetCurTime(), + ReleaseDate: publicDate, Status: 1, RepoVersion: affectBranch} + packId, ipErr := models.InsertOpenGaussPackage(&ogk) + if packId == 0 { + logs.Error("InsertOpenGaussPackage, packId, ipErr: ", packId, ipErr) + } + } + } + } + return nil +} + +func GuassGetDateByGite(pkgList []models.GaussExcelTag, c chan<- []IssueAndPkg, startTime, affectBranch string) { + defer wgTrigger.Done() + owner, token := common.GetOwnerAndToken(int8(2)) + //gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") + st := util.TimeStrToInt(startTime, "2006-01-02") + chData := make([]IssueAndPkg, 0) + for _, v := range pkgList { + rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + // Query the current issues that need to be dealt with + issueTemp, err := models.GetGaussIssueNumber(v.Repo) + if err != nil || issueTemp == nil { + logs.Info("No need to deal with ", v.Repo) + continue + } + for _, isTemp := range issueTemp { + //logs.Info("isTemp===>", isTemp) + var prList []models.PullRequestIssue + prList = getGaussRepoIssueAllPR(affectBranch, token, owner, isTemp.Repo, st, rt, isTemp) + //prList := getGaussRepoIssueAllPR(affectBranch, token, owner, gaussIssuePath, st, rt, isTemp) + //get pull request related issue + repoIssue := make(map[int64]models.PullRequestIssue, 0) + for _, p := range prList { + //getPRRelatedAllIssue(token, owner, v.Repo, st, rt, p.Number, repoIssue) + p.Repo = v.Repo + repoIssue[p.Id] = p + } + if len(repoIssue) > 0 { + chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, + Repo: v.Repo, IssueId: isTemp.IssueId}) + } + } + } + c <- chData +} + +func getGaussRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, + releaseTime int64, isTemp models.IssueTemplate) (prList []models.PullRequestIssue) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests", owner, isTemp.IssueNum) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error("NewRequest, url: ", url, ",err: ", err) + return + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("repo", repo) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error("DefaultClient, url: ", url, ",err: ", err) + return + } + if resp.StatusCode == http.StatusOK { + issuePr := make([]map[string]interface{}, 0) + read, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, url: ", url, ",err: ", err) + return + } + logs.Info("getGaussRepoIssueAllPR, value: ", string(read)) + resp.Body.Close() + err = json.Unmarshal(read, &issuePr) + if err != nil { + logs.Error("Unmarshal, url: ", url, ",err: ", err) + return + } + for _, v := range issuePr { + if _, ok := v["id"]; !ok { + continue + } + pr := models.PullRequestIssue{} + if v["state"].(string) == "merged" && v["mergeable"].(bool) { + //mt := v["closed_at"].(string).(time.Time).Local().Unix() + closedAt := v["closed_at"].(string) + mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") + mergedAt := v["merged_at"].(string) + ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") + //logs.Info("======>>mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo, ", pr: ", v) + //ct := v["merged_at"].(string).(time.Time).Local().Unix() + var pt int64 + if mt > 0 && ct > 0 { + if mt > ct { + pt = ct + } else { + pt = mt + } + if pt >= startTime && pt <= releaseTime { + if v["base"].(map[string]interface{})["label"].(string) == affectBranch { + pr.Id = int64(v["id"].(float64)) + pr.Number = isTemp.IssueNum + pr.CveNumber = isTemp.CveNum + pr.Repo = v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"].(string) + pr.Branch = affectBranch + pr.BrFlag = true + prList = append(prList, pr) + } + } + } + } + } + } else { + resp.Body.Close() + } + return +} + +func downloadGaussPackageFile(localPath, url string) error { + timeStamp := time.Now().Unix() + url = url + "?" + strconv.FormatInt(timeStamp, 10) + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + pkgLock.Lock() + defer pkgLock.Unlock() + if ex, _ := util.IsExistPath(localPath); ex { + err := os.Remove(localPath) + if err != nil { + logs.Error(err) + } + } + out, err := os.Create(localPath) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, resp.Body) + return err + } else { + return errors.New("download file request fail") + } +} + +//ExtractPackageData extract the package data by excel file +func ReadExcelFile(lp string, versions string) (pkgList []models.GaussExcelTag, err error) { + pkgLock.Lock() + defer pkgLock.Unlock() + if lp == "" || path.Ext(lp) != ".xlsx" { + return pkgList, errors.New("the file path is error") + } + f, opErr := excelize.OpenFile(lp) + if opErr != nil { + logs.Error(opErr) + return pkgList, opErr + } + rows, rErr := f.GetRows(versions) + if rErr != nil { + logs.Error(rErr) + return pkgList, rErr + } + for _, row := range rows { + pkgList = append(pkgList, models.GaussExcelTag{PubTime: row[0], Repo: row[1], Packages: row[2]}) + } + return pkgList, nil +} + +// Complete data conversion +func TempCveToGaussCve(tempGaussId int64, ogl []models.OpenGaussCveList) { + if len(ogl) > 0 { + for _, ol := range ogl { + osl := models.OpenGaussCveList{GaussId: tempGaussId, CveNum: ol.CveNum} + cbErr := models.QueryOpenGaussCveByNum(&osl, "GaussId", "CveNum") + if cbErr == nil { + models.UpdateOpenGaussVersionByCid(osl.Id, ol.Id, ol.CveNum) + } + } + models.UpdateOpenGaussPackByGid(tempGaussId, ogl[0].GaussId) + } + models.DeleteOpenGaussCveByGid(tempGaussId) + //models.DeleteOpenGaussPackByGid(tempGaussId) +} + +func DeleteGaussCveVaildData(gaussId int64) { + ogg, oggErr := models.QueryOpenGaussCveByGid(gaussId) + if len(ogg) > 0 { + for _, og := range ogg { + models.DeleteOpenGaussVersionById(og.Id) + } + models.DeleteOpenGaussCveByGid(gaussId) + } else { + logs.Info("QueryOpenGaussCveByGid, oggErr: ", oggErr, ",GaussId: ", gaussId) + } + models.DeleteOpenGaussPackByGid(gaussId) +} + +func ClearHisDataSa(flag int) { + if flag == 2 { + ogl, oglErr := models.QueryOpenGaussSitePreAll() + if len(ogl) == 0 { + logs.Info("ClearHisDataSa, oglErr: ", oglErr) + } else { + for _, o := range ogl { + DeleteGaussCveVaildData(o.GaussId) + } + models.DeleteOpenGaussSitePreAll() + } + } else { + ogg, oggErr := models.QueryOpenGaussCveByGid(0) + if len(ogg) > 0 { + for _, og := range ogg { + models.DeleteOpenGaussVersionById(og.Id) + models.DeleteOpenGaussCveByCid(og.Id) + } + } else { + logs.Info("QueryOpenGaussCveByGid, oggErr: ", oggErr) + } + } + +} + +func PreGeneratedSaData(packName string, bVersionSlice []string, isEque bool, saInitValue *int64) { + if isEque { + ogl, ogErr := models.QuerySaTempByPackName(packName) + if len(ogl) > 0 { + openGaussNum := "openGauss-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.FormatInt(*saInitValue, 10) + var osl models.OpenGaussSiteList + CreateOpenGaussSite(&osl, ogl[0], bVersionSlice, openGaussNum, *saInitValue) + sitGaussId, gErr := models.InsertOpenGaussSiteList(&osl) + if sitGaussId > 0 { + models.UpdateOpenGaussCveByGid(ogl[0].GaussTempId, sitGaussId) + models.UpdateOpenGaussPackByGid(ogl[0].GaussTempId, sitGaussId) + } else { + logs.Error("InsertOpenGaussSiteList, gErr: ", gErr) + return + } + *saInitValue += 1 + // Delete duplicate data + if len(ogl) > 1 { + ogg, oggErr := models.QueryOpenGaussCveByGid(sitGaussId) + if oggErr != nil { + logs.Error("PreGeneratedSaData, QueryOpenGaussCveByGid, ogErr: ", oggErr) + return + } + for _, l := range ogl[1:] { + TempCveToGaussCve(l.GaussTempId, ogg) + } + } + } else { + logs.Error("PreGeneratedSaData, QuerySaTempByPackName, ogErr: ", ogErr) + return + } + } else { + ogl, ogErr := models.QuerySaTempByPackName(packName) + if len(ogl) > 0 { + for _, l := range ogl { + openGaussNum := "openGauss-SA-" + strconv.Itoa(time.Now().Year()) + "-" + strconv.FormatInt(*saInitValue, 10) + var osl models.OpenGaussSiteList + CreateOpenGaussSite(&osl, l, bVersionSlice, openGaussNum, *saInitValue) + sitGaussId, gErr := models.InsertOpenGaussSiteList(&osl) + if sitGaussId > 0 { + models.UpdateOpenGaussCveByGid(l.GaussTempId, sitGaussId) + models.UpdateOpenGaussPackByGid(l.GaussTempId, sitGaussId) + } else { + logs.Error("InsertOpenGaussSiteList, gErr: ", gErr) + return + } + *saInitValue += 1 + } + } else { + logs.Error("PreGeneratedSaData, QuerySaTempByPackName2, ogErr: ", ogErr) + return + } + } + models.DeleteSaTempByPackName(packName) +} + +// Reorganize the data +func CreateOpenGaussSite(osl *models.OpenGaussSiteList, olt models.OpenGaussListTemp, + bVersionSlice []string, openGaussNum string, saInitValue int64) { + osl.SaNum = saInitValue + osl.CveNums = olt.CveNums + osl.PublicDate = olt.PublicDate + osl.InfluenceComponent = olt.InfluenceComponent + osl.Status = 1 + osl.CreateTime = common.GetCurTime() + osl.CveLevelValue = olt.CveLevelValue + osl.CveLevel = olt.CveLevel + osl.Description = olt.Description + if len(bVersionSlice) < 2 { + osl.Theme = olt.Theme + osl.Introduction = olt.Introduction + } else if len(bVersionSlice) == 2 { + bVersionStr := strings.Join(bVersionSlice, " and ") + theme := strings.ReplaceAll(olt.Theme, olt.AffectProduct, bVersionStr) + osl.Theme = theme + introduction := strings.ReplaceAll(olt.Introduction, olt.AffectProduct, bVersionStr) + osl.Introduction = introduction + } else { + bVersionStr := strings.Join(bVersionSlice[:len(bVersionSlice)-1], ",") + " and " + bVersionSlice[len(bVersionSlice)-1] + theme := strings.ReplaceAll(olt.Theme, olt.AffectProduct, bVersionStr) + osl.Theme = theme + introduction := strings.ReplaceAll(olt.Introduction, olt.AffectProduct, bVersionStr) + osl.Introduction = introduction + } + osl.AffectProduct = strings.Join(bVersionSlice, "/") + osl.Summary = olt.Summary + osl.GaussSaNum = openGaussNum + osl.Years = time.Now().Year() +} diff --git a/cve-vulner-manager/taskhandler/grabissue.go b/cve-vulner-manager/taskhandler/grabissue.go new file mode 100644 index 0000000000000000000000000000000000000000..ade0f9a0ac4292db035e20fb65b0f501a6561add --- /dev/null +++ b/cve-vulner-manager/taskhandler/grabissue.go @@ -0,0 +1,287 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/json" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io/ioutil" + "net/http" + "strings" + "sync" +) + +var wg sync.WaitGroup +var issueLock sync.Mutex + +//OrgInfo +type OrgInfo struct { + ID int32 `json:"id,omitempty"` + Login string `json:"login,omitempty"` + URL string `json:"url,omitempty"` + AvatarURL string `json:"avatar_url,omitempty"` + ReposURL string `json:"repos_url,omitempty"` + EventsURL string `json:"events_url,omitempty"` + MembersURL string `json:"members_url,omitempty"` + Description string `json:"description,omitempty"` + Name string `json:"name,omitempty"` + Enterprise string `json:"enterprise,omitempty"` + Members int64 `json:"members,omitempty"` + PublicRepos int64 `json:"public_repos,omitempty"` + PrivateRepos int64 `json:"private_repos,omitempty"` +} + +//Branch Get all branches +type Branch struct { + Name string `json:"name,omitempty"` + Protected bool `json:"protected,omitempty"` + ProtectionURL string `json:"protection_url,omitempty"` +} + +//PackageInfo package info model +type PackageInfo struct { + Code string + Msg string + Data Info +} + +//Info cve info +type Info struct { + Description string +} + +//GrabIssueByOrg grab issue by org name +func GrabIssueByOrg(accToken, org string) error { + logs.Info("Synchronize gitee's issue start......") + orgInfo, err := GetOrgInfo(accToken, org) + if err != nil { + logs.Error("GetOrgInfo, org: ", org, ",err: ", err) + return err + } + reposNum := orgInfo.PublicRepos + orgInfo.PrivateRepos + if reposNum <= 0 { + logs.Info(fmt.Sprintf("%v contain %v repository,grab issue finish!", org, reposNum)) + return errors.New(fmt.Sprintf("%v contain %v repository,grab issue finish!", org, reposNum)) + } + pageSize := reposNum / int64(perPage) + if reposNum%int64(perPage) > 0 { + pageSize = pageSize + 1 + } + var i int64 + for i = 1; i <= pageSize; i++ { + go GetOrgRepos(accToken, org, i) + } + wg.Wait() + logs.Info("Synchronize gitee's issue finish...") + return nil +} + +//GrabIssueByRepo grab issue by repository +func GrabIssueByRepo(accToken, owner, repo, state string) { + page := 1 + product, err := GetInfProduct(accToken, owner, repo) + if err != nil { + logs.Error("GetInfProduct, err: ", err) + } + desc := GetRepoDescription(repo) + for { + list, err := GetIssueList(accToken, owner, repo, state, page) + if err != nil { + logs.Error("GetIssueList, repo: ", repo, ",err: ", err) + break + } + handleIssueList(list, product, desc) + if len(list) < perPage { + break + } + page++ + } +} + +func GetInfProduct(token string, owner string, repo string) (infPro string, err error) { + defer common.Catchs() + resp, err := http.Get(fmt.Sprintf(GiteRepoBranch, owner, repo, token)) + if err != nil { + logs.Error("url: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) + return "", err + } + var branchList []Branch + err = json.Unmarshal(body, &branchList) + if err != nil { + logs.Error("Unmarshal: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) + return "", err + } + affectBranchsxList := []string{} + affectedBranchs := beego.AppConfig.String("cve::affected_branchs") + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + tempBrandList := []string{} + for _, v := range branchList { + if affectBranchsxList != nil && len(affectBranchsxList) > 0 { + for _, affectBrand := range affectBranchsxList { + if strings.HasSuffix(v.Name, affectBrand) { + tempBrandList = append(tempBrandList, v.Name) + break + } + } + } + } + if tempBrandList != nil && len(tempBrandList) > 0 { + infPro = strings.Join(tempBrandList, "/") + } + return +} + +func handleIssueList(list []models.GitIssue, product, desc string) { + if len(list) == 0 { + return + } + //var gil []models.GiteOriginIssue + for _, v := range list { + issueTitle := util.TrimString(v.Title) + issueType := util.TrimString(v.IssueType) + issueNumber := util.TrimString(v.Number) + issueState := util.TrimString(v.State) + issueZhState := util.TrimString(v.IssueState) + repoPath := util.TrimString(v.Repository.Path) + nameSpace := util.TrimString(v.Repository.NameSpace.Path) + organizationID := int8(1) + organizationID = GetOrganizationId(nameSpace) + issueTitle = RegMatchCve(issueTitle) + if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + item := models.GiteOriginIssue{IssueId: v.Id, Url: v.HtmlUrl, + Number: issueNumber, State: issueState, Title: issueTitle, + IssueType: issueType, SecurityHole: v.SecurityHole, + IssueCreateAt: v.CreateAt, IssueUpdateAt: v.UpdateAt, + IssueFinishAt: v.FinishedAt, IssueCreate: v.User.Login, + IssueAssignee: v.Assignee.Login, RepoPath: repoPath, + RepoUrl: v.Repository.Url, InfProduct: product, + RepoDesc: desc, IssueState: issueZhState, + Owner: nameSpace, OrganizationID: organizationID, Status: 0} + //vt := util.TrimString(v.Title) + if strings.HasPrefix(issueTitle, "CVE") { + item.CveNumber = issueTitle + } else if v.Body != "" { + // Use regular expressions to intercept the body and improve it later + sm := util.RegexpCveNumber.FindAllStringSubmatch(v.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + item.CveNumber = util.TrimString(sm[0][1]) + } + } + //vb := util.TrimString(v.Body) + vb := strings.ReplaceAll(v.Body, "：", ":") + item.Body = vb + item.IssueExistTpl = models.IssueExistByNumber(v.Number, v.Id) + //logs.Info("item===> ", item) + err := item.InsertOrUpdate(1) + if err != nil { + logs.Error("insert or update issue fail:", err) + } + } + } +} + +//GetOrgInfo get organization information +func GetOrgInfo(accToken, org string) (OrgInfo, error) { + oi := OrgInfo{} + resp, err := http.Get(fmt.Sprintf(GiteOrgInfoURL, org, accToken)) + if err != nil { + return oi, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return oi, err + } + err = json.Unmarshal(body, &oi) + return oi, err +} + +//GetOrgRepos get organization repository +func GetOrgRepos(accToken, org string, page int64) { + wg.Add(1) + defer wg.Done() + resp, err := http.Get(fmt.Sprintf(GiteOrgReposURL, org, accToken, page, perPage)) + if err != nil { + logs.Error("Get, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + defer resp.Body.Close() + var reps []models.HookRepository + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + err = json.Unmarshal(body, &reps) + if err != nil { + logs.Error("Unmarshal, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + for _, v := range reps { + GrabIssueByRepo(accToken, org, v.Name, "all") + } +} + +//GetIssueList get the repository issue list +func GetIssueList(accToken, owner, repo, state string, page int) (issueList []models.GitIssue, err error) { + giteUrl := fmt.Sprintf(GiteRepoIssuesURL, owner, repo, accToken, state, page, perPage) + resp, err := http.Get(giteUrl) + if err != nil { + logs.Error("Get, GiteRepoIssuesURL: ", giteUrl, ", repo: ", repo, ", err: ", err) + return issueList, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, GiteRepoIssuesURL: ", giteUrl, ", repo: ", repo, ", err: ", err) + return issueList, err + } + //logs.Info("-----------issue list: ", string(body)) + err = json.Unmarshal(body, &issueList) + if err != nil { + logs.Error("Unmarshal, GiteRepoIssuesURL: ", giteUrl, ", repo: ", repo, ", err: ", err) + } + //logs.Info("++++++++++issueList: ", issueList) + return +} + +//GetRepoDescription get repository description +func GetRepoDescription(repo string) (desc string) { + if repo == "" { + return "" + } + url := fmt.Sprintf(RepoInfoURL, repo) + resp, err := http.Get(url) + if err != nil { + logs.Error("Get, RepoInfoURL: ", url, ",err: ", err) + return "" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, RepoInfoURL: ", url, ",err: ", err) + return "" + } + var pkg PackageInfo + err = json.Unmarshal(body, &pkg) + if err != nil { + logs.Error("Unmarshal, RepoInfoURL: ", url, ",err: ", err) + return "" + } + if pkg.Code == "2001" { + return pkg.Data.Description + } + return "" +} diff --git a/cve-vulner-manager/taskhandler/hook.go b/cve-vulner-manager/taskhandler/hook.go new file mode 100644 index 0000000000000000000000000000000000000000..c257395d8598266657b5a6729c568ed9a963232d --- /dev/null +++ b/cve-vulner-manager/taskhandler/hook.go @@ -0,0 +1,337 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/json" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io/ioutil" + "net/http" + "os" + "strings" + "sync" +) + +var wgx sync.WaitGroup + +type HookData struct { + Id int64 + HookUrl string + CreateTime string + Password string + ResultCode int + ProjectId int64 +} + +func ProcHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken string) error { + deleteHook, ok := beego.AppConfig.Int("hook::delete_hook") + if ok != nil { + deleteHook = 1 + } + hookId := int64(0) + count := 5 + for ; ; { + iHook, ihOk := models.GetAllIssueHook(hookId, count, 1) + if ihOk && len(iHook) > 0 { + for _, ih := range iHook { + var hd []HookData + if ih.Owner == gaussOwner { + accessToken = gitGaussToken + } + GetDepositHooks(accessToken, ih.Owner, ih.Repo, &hd) + if len(hd) > 0 { + PrcMutDepositHooks(accessToken, pwd, ih, hd, deleteHook, gaussOwner) + } + } + hookId = iHook[len(iHook)-1].Id + } else { + break + } + } + // Check whether the warehouse is missing + dbRepo := models.QueryOrgAllRepo(owner) + if len(dbRepo) > 0 { + for _, v := range dbRepo { + var hd []HookData + GetDepositHooks(accessToken, owner, v.Path, &hd) + if len(hd) > 0 { + for _, hdx := range hd { + if hdx.HookUrl == hookurl && pwd == hdx.Password { + var localIh models.IssueHooks + localIh.HookUrl = hookurl + localIh.Repo = v.Path + localIh.HookId = hdx.Id + localIh.Owner = owner + PrcMutDepositHooks(accessToken, pwd, localIh, hd, deleteHook, gaussOwner) + } + } + } + } + } + return nil +} + +func GetDepositHooks(accessToken, owner, repo string, hd *[]HookData) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/hooks?access_token=%v&page=1&per_page=50", owner, repo, accessToken) + hookData, err := util.HTTPGet(url) + if err == nil && hookData != nil { + for _, value := range hookData { + if _, ok := value["id"]; !ok { + logs.Error("hookData, err: ", ok, "url: ", url) + continue + } + var hdl HookData + hdl.Id = int64(value["id"].(float64)) + hdl.HookUrl = value["url"].(string) + hdl.CreateTime = value["created_at"].(string) + hdl.Password = value["password"].(string) + hdl.ProjectId = int64(value["project_id"].(float64)) + hdl.ResultCode = int(value["result_code"].(float64)) + *hd = append(*hd, hdl) + } + } +} + +func PrcMutDepositHooks(accessToken, pwd string, ihk models.IssueHooks, + hd []HookData, deleteHook int, gaussOwner string) { + for _, hdk := range hd { + if ihk.HookUrl == hdk.HookUrl && hdk.Password == pwd { + if deleteHook == 2 { + go DeleteRepoHooks(accessToken, ihk.Owner, ihk.Repo, hdk.Id, ihk.Id) + } else { + if hdk.Id != ihk.HookId && ihk.HookId > 0 { + go DeleteDepositHooks(accessToken, ihk.Owner, ihk.Repo, hdk.Id) + } + } + } + } +} + +func DeleteRepoHooks(accessToken, owner, repo string, hookId, ihkId int64) { + DeleteDepositHooks(accessToken, owner, repo, hookId) + models.UpdateIssueHook(ihkId, 2) +} + +func DeleteDepositHooks(accessToken, owner, repo string, hookId int64) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/hooks/%v", owner, repo, hookId) + requestBody := fmt.Sprintf(`{ + "access_token": "%s" + }`, accessToken) + util.HTTPDelCom(url, requestBody) +} + +func AddRepoWebhook(accessToken, owner, path, pwd, hookurl string) error { + var ih models.IssueHooks + ih.CveId = int64(1) + issueId := int64(1) + ih.IssueNum = "000000" + ih.Owner = owner + ih.Status = 1 + ih.Repo = path + ihs, errh := models.GetIssueHook(&ih) + if errh { + hookExist := false + for _, wh := range ihs { + if wh.HookUrl == hookurl { + hookExist = true + break + } + } + if !hookExist { + postErr := PostWebHooks(accessToken, owner, path, hookurl, pwd, ih.IssueNum, issueId, ih.CveId) + if postErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ", postErr: ", postErr) + } + return postErr + } + return nil + } else { + postErr := PostWebHooks(accessToken, owner, path, hookurl, pwd, ih.IssueNum, issueId, ih.CveId) + if postErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ", postErr: ", postErr) + } + return postErr + } +} + +//GetOrgRepos get organization repository +func GetOrgAllRepos(accToken, org, pwd, hookurl string, page int64) { + wgx.Add(1) + defer wgx.Done() + resp, err := http.Get(fmt.Sprintf(GiteOrgReposURL, org, accToken, page, perPage)) + if err != nil { + logs.Error("Get, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + defer resp.Body.Close() + var reps []models.HookRepository + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + err = json.Unmarshal(body, &reps) + if err != nil { + logs.Error("Unmarshal, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + for _, v := range reps { + AddRepoWebhook(accToken, org, v.Name, pwd, hookurl) + } +} + +func GetAllRepoInfo(accessToken, owner, pwd, hookurl string) error { + orgInfo, err := GetOrgInfo(accessToken, owner) + if err != nil { + logs.Error("GetOrgInfo, org: ", owner, ",err: ", err) + return err + } + reposNum := orgInfo.PublicRepos + orgInfo.PrivateRepos + if reposNum <= 0 { + logs.Info(fmt.Sprintf("%v contain %v repository,grab issue finish!", owner, reposNum)) + return errors.New(fmt.Sprintf("%v contain %v repository,grab issue finish!", owner, reposNum)) + } + pageSize := reposNum / int64(perPage) + if reposNum%int64(perPage) > 0 { + pageSize = pageSize + 1 + } + var i int64 + for i = 1; i <= pageSize; i++ { + go GetOrgAllRepos(accessToken, owner, pwd, hookurl, i) + } + wgx.Wait() + return nil +} + +// Create webhook event function entry +func CreateHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken string) error { + repoErr := GetAllRepoInfo(accessToken, owner, pwd, hookurl) + if repoErr == nil { + logs.Info("CreateHookEvent, owner: ", owner, ", webhook created successfully, hookurl: ", hookurl) + } else { + logs.Error("CreateHookEvent, owner: ", owner, ", hookurl: ", hookurl, ",repoErr: ", repoErr) + } + dbRepo := models.QueryOrgAllRepo(owner) + if len(dbRepo) > 0 { + for _, v := range dbRepo { + AddRepoWebhook(accessToken, owner, v.Path, pwd, hookurl) + } + } + return nil +} + +func CheckAffectVerComplete(affectedVersion, packageName, version string, organizateId int8) []string { + affectBranchsxList := make([]string, 0) + affectProductList := make([]string, 0) + unFixValue := make([]string, 0) + owner, accessToken := common.GetOwnerAndToken(organizateId) + if organizateId == 1 || organizateId == 2 { + affectBranchsxList, _ = GetBranchesInfo(accessToken, owner, packageName, organizateId) + } else if organizateId == 3 || organizateId == 4 { + affectBranchsxList = CreateBrandAndTags(accessToken, owner, packageName, organizateId) + } + if len(affectBranchsxList) == 0 { + return unFixValue + } + if affectedVersion != "" && len(affectedVersion) > 1 { + affectProductList = strings.Split(affectedVersion, ",") + } + if len(affectBranchsxList) > 0 { + if len(affectProductList) == 0 { + unFixValue = append(unFixValue, affectBranchsxList...) + } else { + for _, abl := range affectBranchsxList { + flag := false + for _, apl := range affectProductList { + branchSlice := []string{} + if strings.Contains(apl, ":") { + branchSlice = strings.Split(apl, ":") + } else if strings.Contains(apl, "：") { + branchSlice = strings.Split(apl, "：") + } else { + branchSlice = append(branchSlice, apl) + } + if len(branchSlice) == 2 { + affBrand := common.BranchVersionRep(branchSlice[0]) + if strings.TrimSpace(affBrand) == abl { + if len(strings.TrimSpace(branchSlice[1])) > 1 { + if (strings.TrimSpace(branchSlice[1]) == "受影响") || + (strings.TrimSpace(branchSlice[1]) == "不受影响") { + flag = true + } + } + } + } + } + if !flag { + unFixValue = append(unFixValue, abl) + } + } + } + } + logs.Info("The branches that require mandatory verification are: ", unFixValue) + return unFixValue +} + +func QueryEulerRepoOrigin(affectedBranchs, packageName, version string, organizationID int8) string { + affectBranchsxList := make([]string, 0) + unFixValue := make([]string, 0) + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + if len(affectBranchsxList) == 0 { + return affectedBranchs + } + eulerOrgList := models.QueryEulerRepoOriginByNmae(packageName) + if len(eulerOrgList) > 0 { + for _, eol := range eulerOrgList { + if len(eol.Branchs) > 0 { + for _, abl := range affectBranchsxList { + branSlice := strings.Split(eol.Branchs, ",") + for _, bs := range branSlice { + if abl == bs { + unFixValue = append(unFixValue, abl) + break + } + } + } + } + } + } + if len(unFixValue) == 0 { + accessToken := os.Getenv("GITEE_TOKEN") + owner := beego.AppConfig.String("gitee::owner") + branchList, errBrands := GetBranchesInfo(accessToken, owner, packageName, organizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("ProcIssue, Failed to obtain branch information,CveNum: ", + ", path: ", packageName, ", err: ", errBrands) + return "" + } + unFixValue = append(unFixValue, branchList...) + } + if len(unFixValue) > 0 { + valueSlice := removeDuplicateElement(unFixValue) + return strings.Join(valueSlice, ",") + } + return "" +} + +func removeDuplicateElement(element []string) []string { + result := make([]string, 0, len(element)) + temp := map[string]struct{}{} + for _, item := range element { + if _, ok := temp[item]; !ok { + temp[item] = struct{}{} + result = append(result, item) + } + } + return result +} diff --git a/cve-vulner-manager/taskhandler/issue.go b/cve-vulner-manager/taskhandler/issue.go new file mode 100644 index 0000000000000000000000000000000000000000..3ff96a36b082636014e20fa09619856cfd100b89 --- /dev/null +++ b/cve-vulner-manager/taskhandler/issue.go @@ -0,0 +1,186 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "strings" + "time" +) + +// Verify whether the issue on gitee is deleted, and the specific logic is implemented +func ProcIssueIsExists(prnum int) error { + cveId := int64(0) + for { + vc, err := models.QueryAllCveVuln(prnum, cveId) + if err != nil { + logs.Error("QueryAllCveVuln, err: ", err) + break + } + if len(vc) == 0 { + break + } + for _, v := range vc { + cveId = v.CveId + issueTmp := models.IssueTemplate{CveId: v.CveId, Repo: v.PackName, CveNum: v.CveNum} + tempErr := models.GetIssueTemplateByColName(&issueTmp, "CveId", "Repo", "CveNum") + if tempErr != nil { + continue + } + if issueTmp.IssueNum != "" && len(issueTmp.IssueNum) > 0 { + issueErr := error(nil) + issueData := map[string]interface{}{} + path := issueTmp.Repo + owner, accessToken := common.GetOwnerAndToken(v.OrganizationID) + if v.OrganizationID == 2 { + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } + } + issueErr, issueData = GetGiteeIssue(accessToken, owner, path, issueTmp.IssueNum) + if issueErr != nil { + logs.Info("UpdateIssueTemplate1, issueNum: ", issueTmp.IssueNum, issueTmp.StatusName) + issueTmp.Status = 6 + issueTmp.StatusName = "deleted" + issueTmp.DeleteTime = time.Now() + updateErr := models.UpdateIssueTemplate(&issueTmp, "Status", "StatusName", "DeleteTime") + if updateErr != nil { + logs.Error("UpdateIssueTemplate1, err: ", updateErr) + } + } else { + if issueData != nil && issueData["state"] != nil && issueData["state"].(string) != "" { + state := issueData["state"].(string) + if state == "待办的" || state == "开启的" || + strings.ToLower(state) == "open" { + issueTmp.StatusName = "open" + issueTmp.Status = 1 + } else if state == "进行中" || strings.ToLower(state) == "started" || + strings.ToLower(state) == "progressing" { + issueTmp.StatusName = "progressing" + issueTmp.Status = 2 + } else if state == "已完成" || strings.ToLower(state) == "closed" { + issueTmp.StatusName = "closed" + issueTmp.IssueLabel = beego.AppConfig.String("labelFixed") + issueTmp.Status = 3 + } else if state == "已拒绝" || strings.ToLower(state) == "rejected" { + issueTmp.StatusName = "rejected" + issueTmp.Status = 4 + } else if state == "已挂起" || strings.ToLower(state) == "suspended" { + issueTmp.StatusName = "suspended" + issueTmp.Status = 5 + } else { + issueTmp.StatusName = state + } + logs.Info("UpdateIssueTemplate2, issueNum: ", issueTmp.IssueNum, issueTmp.StatusName) + updateErr := models.UpdateIssueTemplate(&issueTmp, "Status", "StatusName", "IssueLabel") + if updateErr != nil { + logs.Error("UpdateIssueTemplate2, err: ", updateErr) + } + } + } + } + } + } + return nil +} + +// Statistics on the creation of openEuler community issues +// Statistics on the creation of openGauss community issues +// Statistics on the creation of issues in the MindSpore community +func StatistCommunityIssue(cyc models.CommunityYamlConfig) { + // cve credibility level + credibilityLevel, ok := beego.AppConfig.Int("cve::credibility_level") + if ok != nil { + logs.Error("cve::credibility_level, err:", ok) + credibilityLevel = 3 + } + id := int64(0) + count := 50 + for { + communityPackage := models.GetCommunityYamlList(cyc.TableName, id, count) + if len(communityPackage) > 0 { + for _, cpg := range communityPackage { + id = cpg.Id + if cyc.OrganizationID == 1 && len(cpg.RepoName) < 1 { + cpg.RepoName = cpg.PackageName + } + ous := models.QueryOriginUpstreamByName(cpg.PackageName, credibilityLevel) + CheckOriginUpstreamToCenter(ous, cpg.PackageName, cpg.RepoName, cpg.PackageVersion, cyc.OrganizationID) + } + } else { + break + } + } +} + +func CheckOriginUpstreamToCenter(ous []models.OriginUpstream, + packageName, repoName, packageVersion string, organizationID int8) { + for _, ou := range ous { + ics := models.IssueCommunityStatistics{CveNum: ou.CveNum, Repo: repoName, ExistFlag: organizationID} + queryErr := models.QueryIssueCommunityStatistics(&ics, "CveNum", "Repo", "ExistFlag") + if ics.Id > 0 { + continue + } else { + logs.Info("QueryIssueCommunityStatistics, queryErr: ", queryErr) + } + packNameMap := map[string]string{} + packNameList := []string{} + if len(ou.PackName) > 0 { + packNameList = strings.Split(ou.PackName, ",") + if len(packNameList) > 0 { + for _, pk := range packNameList { + pkList := strings.Split(pk, "==") + if len(pkList) == 2 { + if _, ok := packNameMap[pkList[0]]; ok { + versionList := strings.Split(packNameMap[pkList[0]], ",") + verExist := false + for _, vl := range versionList { + if vl == pkList[1] { + verExist = true + break + } + } + if !verExist { + packNameMap[pkList[0]] += "," + pkList[1] + } + } else { + packNameMap[pkList[0]] = pkList[1] + } + } + } + } + } + for key, value := range packNameMap { + if key != "" && len(key) > 1 { + pkList := []string{} + pkList = append(pkList, key) + versionList := make([]string, 0) + if value != "" && len(value) > 0 { + versionList = strings.Split(value, ",") + if len(versionList) > 0 { + _, ok := common.FindSliceEm(versionList, packageVersion) + if key == packageName && ok { + cveRes, cveOk := models.QueryCveByNum(ou.CveNum, repoName, organizationID) + if cveOk && cveRes.Status == 2 { + ics := models.IssueCommunityStatistics{CveId: cveRes.CveId, + CveNum: cveRes.CveNum, Repo: repoName, PackageName: packageName, + Version: value, ExistFlag: organizationID, CreateTime: common.GetCurTime()} + num, inErr := models.InsertIssueCommunityStatistics(&ics) + if num == 0 { + logs.Error("InsertIssueCommunityStatistics, inErr: ", inErr, num) + } + } else { + ou.Status = 1 + upErr := models.UpdateOriginUpstream(&ou, "Status") + if upErr != nil { + logs.Error("UpdateOriginUpstream, upErr: ", upErr) + } + } + } + } + } + } + } + } +} diff --git a/cve-vulner-manager/taskhandler/issuestatistics.go b/cve-vulner-manager/taskhandler/issuestatistics.go new file mode 100644 index 0000000000000000000000000000000000000000..cd8ec5d8f7ef5e3f2215d17ffbd7af5f9c31fab4 --- /dev/null +++ b/cve-vulner-manager/taskhandler/issuestatistics.go @@ -0,0 +1,374 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "errors" + "fmt" + "github.com/360EntSecGroup-Skylar/excelize/v2" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "path/filepath" + "strconv" +) + +const sheetName = "CVE_list" + +type IssueStr struct { + cveCount int64 + HighCveCount int64 + CurIndex int64 +} + +func GetIssueData(beforeMonth, prcnum int, templateId int64) ([]models.IssueTemplate, error) { + beforeTime := common.GetBeforeDate(2, beforeMonth) + it, err := models.QueryIssueTemplateByTime(beforeTime, prcnum, templateId, 3) + return it, err +} + +func GetNoLinkIssueData(beforeDate, prcnum int, templateId int64) ([]models.IssueTemplate, error) { + beforeTime := common.GetBeforeTime(beforeDate) + it, err := models.QueryIssueTemplateByLink(beforeTime, prcnum, templateId, 3) + return it, err +} + +func createExcel(beforeMonth int) (string, string) { + beforeTime := common.GetBeforeDate(2, beforeMonth) + // File storage directory + dir := beego.AppConfig.String("fileDir") + common.CreateDir(dir) + excelName := "" + if len(beforeTime) > 10 { + excelName = "CVE_ISSUE_" + beforeTime[:10] + "_" + common.GetCurDate() + ".xlsx" + } else { + excelName = "CVE_ISSUE_" + beforeTime + "_" + common.GetCurDate() + ".xlsx" + } + excelPath := filepath.Join(dir, excelName) + xlsx := excelize.NewFile() + index := xlsx.NewSheet(sheetName) + sheetTileMap := make(map[string]string) + sheetTileMap["A1"] = "序号" + sheetTileMap["B1"] = "issueID" + sheetTileMap["C1"] = "issue标题" + sheetTileMap["D1"] = "issue责任人" + sheetTileMap["E1"] = "issue状态" + sheetTileMap["F1"] = "issue创建时间" + sheetTileMap["G1"] = "issue所属仓库" + sheetTileMap["H1"] = "CVSS评分" + sheetTileMap["I1"] = "评分级别" + for k, v := range sheetTileMap { + xlsx.SetCellValue(sheetName, k, v) + } + xlsx.SetActiveSheet(index) + err := xlsx.SaveAs(excelPath) + if err != nil { + logs.Error(err) + return "", "" + } + return excelPath, dir +} + +func CveCredExcel() (string, string) { + // File storage directory + dir := beego.AppConfig.String("fileDir") + common.CreateDir(dir) + excelName := "CVE_Low_Credibility_" + common.GetCurDate() + ".xlsx" + excelPath := filepath.Join(dir, excelName) + xlsx := excelize.NewFile() + index := xlsx.NewSheet(sheetName) + sheetTileMap := make(map[string]string) + sheetTileMap["A1"] = "CVE编号" + sheetTileMap["B1"] = "CVE影响的组件和版本" + sheetTileMap["C1"] = "CVE描述" + sheetTileMap["D1"] = "CVE可信等级(4:包名版本号通过SA获取、版本号为修复版本号、修复版本以下都视为受影响版本;5:包名版本号通过SA获取并通过别名匹配、版本号为修复版本号、修复版本以下都视为受影响版本;6:版本号未匹配、包名通过以上任意方式匹配、可信度最低)" + sheetTileMap["E1"] = "CVE发布时间" + for k, v := range sheetTileMap { + xlsx.SetCellValue(sheetName, k, v) + } + xlsx.SetActiveSheet(index) + err := xlsx.SaveAs(excelPath) + if err != nil { + logs.Error(err) + return "", "" + } + return excelPath, dir +} + +func procCredData(org models.OriginUpstream, desc models.OriginUpstreamDesc) []interface{} { + cveData := make([]interface{}, 0) + cveData = append(cveData, org.CveNum) + cveData = append(cveData, org.PackName) + cveData = append(cveData, desc.EnDescription) + cveData = append(cveData, org.Credibility) + cveData = append(cveData, org.PublishedDate) + return cveData +} + +func ReadWriteExcel(excelPath, owner string, iss []models.IssueTemplate, is *IssueStr) (int64, error) { + file, openErr := excelize.OpenFile(excelPath) + if openErr != nil { + logs.Error("fail to open the file, excelPath: ", excelPath, ", openErr: ", openErr) + return iss[len(iss)-1].TemplateId, openErr + } + var templateId = int64(0) + for _, its := range iss { + templateId = its.TemplateId + cveData := procIssueData(its, is, owner) + if len(cveData) > 0 { + rows, sheetErr := file.GetRows(sheetName) + if sheetErr != nil { + logs.Error(sheetErr) + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + setErr := file.SetSheetRow(sheetName, axis, &cveData) + if setErr != nil { + logs.Error("setErr: ", setErr) + } + } + } + fileErr := file.SaveAs(excelPath) + if fileErr != nil { + logs.Error("Failed to save file, ", fileErr) + } + return templateId, fileErr +} + +func procIssueData(its models.IssueTemplate, is *IssueStr, owner string) []interface{} { + cveData := make([]interface{}, 0) + if its.Owner != owner || len(its.IssueNum) < 2 { + return cveData + } + is.CurIndex += 1 + is.cveCount += 1 + cveData = append(cveData, is.CurIndex) + cveData = append(cveData, its.IssueNum) + cveData = append(cveData, its.CveNum) + cveData = append(cveData, its.Assignee) + cveData = append(cveData, IssueStateConversion(its.Status)) + cveData = append(cveData, common.TimeConverStr(its.CreateTime.String()[:19])) + cveData = append(cveData, its.Owner+"/"+its.OwnedComponent) + cveData = append(cveData, its.NVDScore) + cveData = append(cveData, models.OpenEulerScoreProc(its.NVDScore)) + if its.NVDScore >= 7 { + is.HighCveCount += 1 + } + return cveData +} + +func QueryCveDesc(cveId int64) models.OriginUpstreamDesc { + var desc models.OriginUpstreamDesc + desc.CveId = cveId + ordErr := models.GetOrgCveDesc(&desc, "CveId") + if ordErr != nil { + logs.Error("Failed to query data, GetOrgCveDesc, err: ", ordErr) + } + return desc +} + +func ReadWriteCredExcel(excelPath string, org []models.OriginUpstream) (int64, error) { + file, openErr := excelize.OpenFile(excelPath) + if openErr != nil { + logs.Error("fail to open the file, ", excelPath) + return org[len(org)-1].CveId, openErr + } + tempCveId := int64(0) + for _, cred := range org { + tempCveId = cred.CveId + desc := QueryCveDesc(cred.CveId) + cveData := procCredData(cred, desc) + if len(cveData) > 0 { + rows, sheetErr := file.GetRows(sheetName) + if sheetErr != nil { + logs.Error(sheetErr) + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + setErr := file.SetSheetRow(sheetName, axis, &cveData) + if setErr != nil { + logs.Error("setErr: ", setErr) + } + } + models.UpdateOriginStatus(common.GetCurTime(), cred.PackName, cred.Version, cred.CveId, 6) + } + fileErr := file.SaveAs(excelPath) + if fileErr != nil { + logs.Error("Failed to save file, ", fileErr) + } + return tempCveId, fileErr +} + +func pressFileZip(excelPath, dir string) string { + totalFileSlice := make([]string, 0) + totalFileSlice = append(totalFileSlice, excelPath) + zipFileName := "CVE_ISSUE_" + common.GetCurDate() + ".zip" + zipFileName = filepath.Join(dir, zipFileName) + zipErr := ZipFiles(zipFileName, totalFileSlice, dir, dir) + if zipErr != nil { + logs.Error("File compression failed: err: ", zipErr) + } + return zipFileName +} + +func IssueStatistics(beforeMonth, prcnum int, owner string) error { + excelPath, _ := createExcel(beforeMonth) + if excelPath == "" { + logs.Error("Failed to create file") + return errors.New("Failed to create file") + } + fileSlice := make([]string, 0) + templateId := int64(0) + var is IssueStr + for { + it, err := GetIssueData(beforeMonth, prcnum, templateId) + if err != nil { + logs.Error("GetIssueData, err: ", err) + break + } + logs.Info("it==>", it) + if len(it) == 0 { + break + } + templateId, err = ReadWriteExcel(excelPath, owner, it, &is) + if err != nil { + templateId = 0 + break + } + } + zipFileName := excelPath + if is.cveCount > 0 { + if templateId > 0 { + cBody := fmt.Sprintf("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" + + strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 7分以上CVE请在周三之前解决. \r\n" + + "已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n") + sendError := SendEmail(zipFileName, 2, cBody, "") + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + } + } + } + fileSlice = append(fileSlice, excelPath) + fileSlice = append(fileSlice, zipFileName) + DelFile(fileSlice) + return nil +} + +func CveCredibilityStatistics(beforeDate, prcnum int) error { + credibilityLevel, ok := beego.AppConfig.Int("cve::credibility_level") + if ok != nil { + credibilityLevel = 3 + } + beforeTime := common.GetSpecialDate(beforeDate) + excelPath, _ := CveCredExcel() + if excelPath == "" { + logs.Error("Failed to create file") + return errors.New("Failed to create file") + } + tempCveId := int64(0) + fileSlice := make([]string, 0) + for { + org, num, err := models.QueryLowCredibilityCve(beforeTime, prcnum, credibilityLevel, tempCveId) + if len(org) > 0 { + tempCveId, err = ReadWriteCredExcel(excelPath, org) + if err != nil { + tempCveId = 0 + break + } + } else { + logs.Error("failed: ", num, ", err: ", err) + break + } + } + zipFileName := excelPath + if tempCveId > 0 { + //zipFileName := pressFileZip(excelPath, dir) + cBody := fmt.Sprintf("hi all: \r\n 当前中科院上传的可信度为4/5/6等级的数据, " + + "请见附件! 如果有需要提交issue的CVE, 请整理成人工提交CVE的excel格式, 工具进行处理. \r\n") + subject := "可信度较低的cve漏洞反馈" + sendError := SendEmail(zipFileName, 3, cBody, subject) + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + return sendError + } + } + fileSlice = append(fileSlice, excelPath) + fileSlice = append(fileSlice, zipFileName) + DelFile(fileSlice) + return nil +} + +func ProcSecLinkTemplate(beforeDate, prcnum int, owner, accessToken string) error { + secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") + templateId := int64(0) + for { + it, err := GetNoLinkIssueData(beforeDate, prcnum, templateId) + if err != nil { + logs.Error("GetNoLinkIssueData, err: ", err) + break + } + logs.Info("it==>", it) + if len(it) == 0 { + break + } + for _, temp := range it { + templateId = temp.TemplateId + cveCenter := models.VulnCenter{CveId: temp.CveId, CveNum: temp.CveNum} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if cveErr != nil || cveCenter.OrganizationID != 1 { + continue + } + // Determine whether cve has been processed + issueExist, saData := GetCveSecurityNotice(temp.CveNum, temp.Repo) + if issueExist && len(saData.Result.AffectedProduct) > 2 { + // Update sa release time + UpdateSAReleaseTime(saData.Result.AffectedProduct, saData.Result.CreateTime, temp.TemplateId) + secLink := secLinkConfig + "/zh/security/safety-bulletin/detail.html?id=" + saData.Result.AffectedProduct + if temp.SecLink != secLink { + temp.SecLink = secLink + } else { + continue + } + _, issueErr := UpdateIssueToGit(accessToken, owner, temp.Repo, + cveCenter, temp) + logs.Info("ProcSecLinkTemplate1, UpdateIssueToGit, err: ", issueErr) + // release + temp.IssueStatus = 5 + tempErr := models.UpdateIssueTemplate(&temp, "SecLink", "IssueStatus") + logs.Info("ProcSecLinkTemplate1, UpdateIssueTemplate, err: ", tempErr) + } else { + if len(temp.SecLink) > 2 { + temp.SecLink = "" + _, issueErr := UpdateIssueToGit(accessToken, owner, temp.Repo, + cveCenter, temp) + logs.Info("ProcSecLinkTemplate2, UpdateIssueToGit, err: ", issueErr) + // release + temp.IssueStatus = 5 + tempErr := models.UpdateIssueTemplate(&temp, "SecLink", "IssueStatus") + logs.Info("ProcSecLinkTemplate2, UpdateIssueTemplate, err: ", tempErr) + } + } + } + } + return nil +} + +// Update the release time of sa +func UpdateSAReleaseTime(saNum, saReleaseTime string, templateId int64) { + // query issue + ita := models.IssueTemplateAssociation{TemplateId: templateId} + tempErr := models.QueryIssueTemplateAssociation(&ita, "templateId") + if tempErr == nil { + ita.SaReleaseTime = saReleaseTime + updateErr := models.UpdateIssueTemplateAssociation(&ita, "SaReleaseTime") + if updateErr != nil { + logs.Error("UpdateIssueTemplateAssociation, err: ", updateErr) + } + } else { + ita.TemplateId = templateId + ita.SaReleaseTime = saReleaseTime + _, insertErr := models.InsertIssueTemplateAssociation(&ita) + if insertErr != nil { + logs.Error("InsertIssueTemplateAssociation, err: ", insertErr) + } + } +} diff --git a/cve-vulner-manager/taskhandler/obs.go b/cve-vulner-manager/taskhandler/obs.go new file mode 100644 index 0000000000000000000000000000000000000000..469daf3e634e29d983f4a5548bf643f72e42e318 --- /dev/null +++ b/cve-vulner-manager/taskhandler/obs.go @@ -0,0 +1,98 @@ +package taskhandler + +import ( + "cvevulner/obs" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" +) + +func ObsInit() *obs.ObsClient { + var ak = beego.AppConfig.String("obs::access_key_id") + var sk = beego.AppConfig.String("obs::secret_access_key") + var endpoint = beego.AppConfig.String("obs::endpoint") + var obsClient, _ = obs.New(ak, sk, endpoint) + return obsClient +} + +func ObsDeleteObject(obsDir string) { + input := &obs.DeleteObjectInput{} + input.Bucket = beego.AppConfig.String("obs::bucket") + input.Key = obsDir + obsClient := ObsInit() + output, err := obsClient.DeleteObject(input) + if err == nil { + logs.Info("ObsDeleteObject, obsDir: ",obsDir, ",RequestId:%s\n", output.RequestId) + } else if obsError, ok := err.(obs.ObsError); ok { + logs.Error("ObsDeleteObject, obsDir: ",obsDir, ",Code:%s\n", obsError.Code) + logs.Error("ObsDeleteObject, obsDir: ",obsDir, ",Message:%s\n", obsError.Message) + } +} + +func ObsCreateDir(obsDir string) (error, string) { + ObsDeleteObject(obsDir) + input := &obs.PutObjectInput{} + input.Bucket = beego.AppConfig.String("obs::bucket") + input.Key = obsDir + obsClient := ObsInit() + output, err := obsClient.PutObject(input) + if err == nil { + logs.Info("ObsCreateDir, obsDir: ",obsDir, ",obs, RequestId:%s\n", output.RequestId) + } else if obsError, ok := err.(obs.ObsError); ok { + logs.Error("ObsCreateDir, obsDir: ",obsDir, ",obs, Code:%s\n", obsError.Code) + logs.Error("ObsCreateDir, obsDir: ",obsDir, ",obs, Message:%s\n", obsError.Message) + } + return err, obsDir +} + +func ObsUploadFile(objectName, localfile string) error { + input := &obs.PutFileInput{} + input.Bucket = beego.AppConfig.String("obs::bucket") + input.Key = objectName + input.SourceFile = localfile + obsClient := ObsInit() + output, err := obsClient.PutFile(input) + if err == nil { + logs.Info("objectName: ",objectName, ",RequestId:%s\n", output.RequestId) + logs.Info("ETag:%s\n", output.ETag) + } else if obsError, ok := err.(obs.ObsError); ok { + logs.Error("objectName: ",objectName, ",Code:%s\n", obsError.Code) + logs.Error("objectName: ",objectName, ",Message:%s\n", obsError.Message) + } + return err +} + +func ObsDownloadFile(objectName, localfile string) error { + input := &obs.GetObjectInput{} + input.Bucket = beego.AppConfig.String("obs::bucket") + downDir := beego.AppConfig.String("obs::download_parent_directory") + input.Key = downDir + objectName + obsClient := ObsInit() + output, err := obsClient.GetObject(input) + if err != nil { + logs.Error("ObsDownloadFile, obsClient.GetObject, err: ", err) + return err + } + defer func() { + errMsg := output.Body.Close() + if errMsg != nil { + logs.Error("ObsDownloadFile, output.Body.Close, errMsg: ", errMsg) + } + }() + logs.Info("Object content:") + body, err := ioutil.ReadAll(output.Body) + if err != nil { + logs.Error("ObsDownloadFile, ReadAll, err: ", err) + return err + } + logs.Info(string(body)) + f, fErr := os.Create(localfile) + if fErr != nil { + logs.Error("ObsDownloadFile, fErr: ", fErr) + return fErr + } + defer f.Close() + _, err = f.Write(body) + return nil +} diff --git a/cve-vulner-manager/taskhandler/oricvecheck.go b/cve-vulner-manager/taskhandler/oricvecheck.go new file mode 100644 index 0000000000000000000000000000000000000000..1cc08fa3e8dd3bd4392264b2feba5b8e498fcb8f --- /dev/null +++ b/cve-vulner-manager/taskhandler/oricvecheck.go @@ -0,0 +1,168 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "github.com/astaxie/beego/logs" + "strconv" + "strings" + "time" +) + +func CheckCveOriginData(prcnum int) (string, error) { + defer common.Catchs() + //var os []models.OriginUpstream + count := 0 + ok := models.UpdateOriginExistTemp() + if !ok { + logs.Info("UpdateOriginExistTemp, No data to process") + return "", nil + } + for { + os, num, err := models.QueryOriginCveE(prcnum) + if err != nil || num == 0 { + logs.Info("Currently no abnormal cve raw data processing, err: ", err, + ", Processing time range: beforetime: ", + "curtime: ", common.GetCurTime()) + break + } + ch := make(chan int, len(os)) + for i, cveOrg := range os { + count = count + 1 + go func(idx int, cveData models.OriginUpstream) { + ch <- idx + packNameList := []string{} + if cveData.PackName != "" && len(cveData.PackName) > 0 { + packNameList = strings.Split(cveData.PackName, ",") + } + if len(packNameList) > 0 { + for _, pk := range packNameList { + pkList := strings.Split(pk, "==") + if len(pkList) == 2 { + var gits models.GitOpenEuler + gits.PackageName = pkList[0] + gits.Version = pkList[1] + gits.Status = 1 + ok := models.QueryCveOpeneulerdata(&gits) + if !ok { + opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("Data added to the cve vulnerability: ", cveData, ", gits: ", gits) + continue + } + ms := models.MindSporeYaml{PackageName: pkList[0], Version: pkList[1]} + sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") + if ms.Id > 0 { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("Correspondence between warehouse and version is wrong, restore: ", cveData, ", gits: ", gits, sporeErr) + continue + } + opky := models.OpenLookengYaml{PackageName: pkList[0], Version: pkList[1]} + opkyErr := models.GetOpenLookengYaml(&opky, "PackageName", "Version") + if opky.Id > 0 { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("Correspondence between warehouse and version is wrong, restore: ", cveData, ", gits: ", gits, opkyErr) + continue + } + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 0) + } else { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("Data added to the cve vulnerability:", cveData, ", gits: ", gits) + } + } + } + } else { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 0) + logs.Info("Data error, field is empty, PackName: ", cveData.PackName, + ",Version: ", cveData.Version, ", CveId, ", cveData.CveId) + } + }(i, cveOrg) + } + for i := 0; i < cap(ch); i++ { + <-ch + } + close(ch) + } + return "", nil +} + +func UpdateAbnCveStatus(prcNum, days int, cveSt string) (string, error) { + beforeTime := common.GetBeforeTime(days) + cveId := int64(0) + for { + cveData, err := models.QueryAbnCve(cveSt, beforeTime, prcNum, cveId) + if err == nil && len(cveData) > 0 { + for _, cd := range cveData { + cveId = cd.CveId + issueTmp := models.IssueTemplate{CveNum: cd.CveNum, CveId: cd.CveId, Repo: cd.PackName} + err := models.GetIssueTemplateByColName(&issueTmp, "CveNum", "CveId", "Repo") + if issueTmp.TemplateId < 1 { + cd.Status = 0 + update := models.UpdateVulnCenter(&cd, "Status") + if !update { + logs.Error("update vulnCenter status fail ", cd, ", err: ", err) + continue + } + } else { + if issueTmp.IssueId < 1 || issueTmp.IssueNum == "" || len(issueTmp.IssueNum) < 2 { + _, oks := models.QueryIssueCveByNum(cd.CveNum, cd.PackName, cd.OrganizationID) + if oks { + continue + } + models.DeleteIssueTemplate(issueTmp.TemplateId) + cd.Status = 0 + update := models.UpdateVulnCenter(&cd, "Status") + if !update { + logs.Error("update vulnCenter status fail ", cd, ", err: ", err) + continue + } + } + } + } + } else { + break + } + } + return "", nil +} + +func AddSaNumber(saReAmount int64) (string, error) { + totalSaReAmount := 2 * saReAmount + curYears := strconv.Itoa(time.Now().Year()) + saNum := int64(1001) + models.DeleteSaNumberByYears(curYears, 1) + var count int64 + count = models.GetSaNumCountByYears(curYears, 1) + if count <= saReAmount { + var sa models.SaNumber + getErr := models.GetSaNumberByYears(&sa, curYears, 0, 2) + logs.Info("AddSaNumber, getErr: ", getErr) + if sa.SaId == 0 { + for i := int64(0); i < totalSaReAmount; i++ { + var localSa models.SaNumber + localSa.OpenEulerSANum = "openEuler-SA-" + curYears + "-" + strconv.FormatInt(saNum, 10) + localSa.Status = 1 + localSa.SaNum = saNum + localSa.SaYears = curYears + localSa.CreateTime = common.GetCurTime() + models.InsertSaNumber(&localSa) + saNum += 1 + } + } else { + saNum = sa.SaNum + 1 + for i := int64(0); i < totalSaReAmount-count; i++ { + var localSa models.SaNumber + localSa.OpenEulerSANum = "openEuler-SA-" + curYears + "-" + strconv.FormatInt(saNum, 10) + localSa.Status = 1 + localSa.SaNum = saNum + localSa.SaYears = curYears + localSa.CreateTime = common.GetCurTime() + models.InsertSaNumber(&localSa) + saNum += 1 + } + } + } + return "", nil +} diff --git a/cve-vulner-manager/taskhandler/sendemail.go b/cve-vulner-manager/taskhandler/sendemail.go new file mode 100644 index 0000000000000000000000000000000000000000..2e3f3300920935dce81c480f216551008c9aec3a --- /dev/null +++ b/cve-vulner-manager/taskhandler/sendemail.go @@ -0,0 +1,378 @@ +package taskhandler + +import ( + "archive/zip" + "bytes" + "cvevulner/common" + "cvevulner/models" + "encoding/base64" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "gopkg.in/gomail.v2" + "io" + "io/ioutil" + "mime" + "net/smtp" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// define email interface, and implemented auth and send method +type Mail interface { + Auth() + Send(message Message) error +} + +type SendMail struct { + user string + password string + host string + port string + auth smtp.Auth +} + +type Attachment struct { + name string + contentType string + withFile bool +} + +type Message struct { + from string + to []string + cc []string + bcc []string + subject string + body string + contentType string + attachment Attachment +} + +// srcFile could be a single file or a directory +// ZipFiles compresses one or many files into a single zip archive file. +func ZipFiles(filename string, files []string, oldform, newform string) error { + newZipFile, err := os.Create(filename) + if err != nil { + return err + } + defer newZipFile.Close() + zipWriter := zip.NewWriter(newZipFile) + defer zipWriter.Close() + for _, file := range files { + fisExist, _ := PathExists(file) + if !fisExist { + logs.Error("ZipFiles, not exist, file: ", file) + continue + } + zipfile, err := os.Open(file) + if err != nil { + return err + } + defer zipfile.Close() + info, err := zipfile.Stat() + if err != nil { + return err + } + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + header.Name = strings.Replace(file, oldform, newform, -1) + header.Method = zip.Deflate + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + if _, err = io.Copy(writer, zipfile); err != nil { + return err + } + } + return nil +} + +func SendEmail(attchStr string, flag int, cBody, subject string) error { + //var mail Mail + //emailName := beego.AppConfig.String("email::email_name") + //emailPwd := beego.AppConfig.String("email::email_pwd") + //emailHost := beego.AppConfig.String("email::email_host") + //emailPort := beego.AppConfig.String("email::email_port") + //emailPwd = "" + elt := models.EmailList{EmailType: 1} + el, eltErr := elt.Read("EmailType") + if eltErr != nil { + logs.Error("Failed to get mailing list, err: ", eltErr) + return eltErr + } + toEmailName := []string{} + for _, em := range el { + toEmailName = append(toEmailName, em.EmailName) + } + //_, attchName := filepath.Split(attchStr) + emailError := error(nil) + //mail = &SendMail{user: emailName, password: emailPwd, host: emailHost, port: emailPort} + if flag == 1 { + //message := Message{from: emailName, + // to: toEmailName, + // cc: []string{}, + // bcc: []string{}, + // subject: "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime(), + // body: fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been " + + // "uploaded to the file server is as follows: \r\n" + strings.ReplaceAll(attchStr, "download/", "")), + // contentType: "text/plain;charset=utf-8", + // attachment: Attachment{ + // name: attchStr, + // contentType: "text/plain", + // withFile: true, + // }, + //} + //emailError = mail.Send(message) + //if emailError == nil { + // logs.Info("Notify cvrf that the email was sent successfully!") + //} else { + // logs.Error("Notify cvrf mail delivery failure!") + //} + // + subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime() + body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been " + + "uploaded to the file server is as follows: \r\n" + strings.ReplaceAll(attchStr, "download/", "")) + emailCc := []string{} + sendErr := SendCommonMail(toEmailName, emailCc, subject, body, attchStr) + if sendErr == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } + } else if flag == 2 { + ism, queryErr := models.QueryIssueStatisticEmail() + if len(ism) > 0 { + toEmail := make([]string, 0) + ccEmail := make([]string, 0) + for _, im := range ism { + if im.EmailType == 1 { + toEmail = append(toEmail, im.EmailName) + } else { + ccEmail = append(ccEmail, im.EmailName) + } + } + curDate := common.GetCurDateFormat() + //message := Message{from: emailName, + // to: toEmail, + // cc: ccEmail, + // bcc: []string{}, + // subject: "【" + curDate + " CVE进展】openEuler需解决CVE列表，请各接口人尽快安排解决", + // body: cBody, + // contentType: "text/plain;charset=utf-8", + // attachment: Attachment{ + // name: attchStr, + // contentType: "text/plain", + // withFile: true, + // }, + //} + //emailError = mail.Send(message) + //if emailError == nil { + // logs.Info("Notify issue statistics that the email was sent successfully!") + //} else { + // logs.Error("Notify issue statistics mail delivery failure!") + //} + subject := "【" + curDate + " CVE进展】openEuler需解决CVE列表，请各接口人尽快安排解决" + body := cBody + sendErr := SendCommonMail(toEmail, ccEmail, subject, body, attchStr) + if sendErr == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } + } else { + emailError = queryErr + } + } else if flag == 3 { + //message := Message{from: emailName, + // to: toEmailName, + // cc: []string{}, + // bcc: []string{}, + // subject: subject, + // body: cBody, + // contentType: "text/plain;charset=utf-8", + // attachment: Attachment{ + // name: attchStr, + // contentType: "text/plain", + // withFile: true, + // }, + //} + //emailError = mail.Send(message) + //if emailError == nil { + // logs.Info("Notify cve that the email was sent successfully!") + //} else { + // logs.Error("Notify cve mail delivery failure!") + //} + subject := subject + body := cBody + sendErr := SendCommonMail(toEmailName, []string{}, subject, body, attchStr) + if sendErr == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } + } else { + //message := Message{from: emailName, + // to: toEmailName, + // cc: []string{}, + // bcc: []string{}, + // subject: "updateinfo.xml and the excel file corresponding to SA", + // body: "hi all: \r\n The attachment is: updateinfo.xml and the excel file corresponding to SA.", + // contentType: "text/plain;charset=utf-8", + // attachment: Attachment{ + // name: attchStr, + // contentType: "text/plain", + // withFile: true, + // }, + //} + //emailError = mail.Send(message) + //if emailError == nil { + // logs.Info("updateinfo.xml and the excel file corresponding to SA, Mail sent successfully!") + //} else { + // logs.Error("updateinfo.xml and the excel file corresponding to SA, Mail sent failure!") + //} + subject := "updateinfo.xml and the excel file corresponding to SA" + body := "hi all: \r\n The attachment is: updateinfo.xml and the excel file corresponding to SA." + sendErr := SendCommonMail(toEmailName, []string{}, subject, body, attchStr) + if sendErr == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } + } + return emailError +} + +func (mail *SendMail) Auth() { + mail.auth = smtp.PlainAuth("", mail.user, mail.password, mail.host) +} + +func (mail SendMail) Send(message Message) error { + mail.Auth() + buffer := bytes.NewBuffer(nil) + boundary := "GoBoundary" + Header := make(map[string]string) + //Header["From"] = message.from + Header["From"] = "cve-manager" + "<" + message.from + ">" + Header["To"] = strings.Join(message.to, ";") + Header["Cc"] = strings.Join(message.cc, ";") + Header["Bcc"] = strings.Join(message.bcc, ";") + Header["Subject"] = message.subject + Header["Content-Type"] = "multipart/mixed;boundary=" + boundary + Header["Mime-Version"] = "1.0" + Header["Date"] = time.Now().String() + mail.writeHeader(buffer, Header) + body := "\r\n--" + boundary + "\r\n" + body += "Content-Type:" + message.contentType + "\r\n" + body += "\r\n" + message.body + "\r\n" + buffer.WriteString(body) + if message.attachment.withFile { + attachment := "\r\n--" + boundary + "\r\n" + attachment += "Content-Transfer-Encoding:base64\r\n" + attachment += "Content-Disposition:attachment\r\n" + attachment += "Content-Type:" + message.attachment.contentType + ";name=\"" + + strings.Replace(message.attachment.name, "download/", "", -1) + "\"\r\n" + buffer.WriteString(attachment) + defer func() { + if err := recover(); err != nil { + logs.Error(err) + } + }() + mail.writeFile(buffer, message.attachment.name) + } + buffer.WriteString("\r\n--" + boundary + "--") + toSend := make([]string, 0) + toSend = append(toSend, message.to...) + toSend = append(toSend, message.cc...) + header := smtp.SendMail(mail.host+":"+mail.port, mail.auth, message.from, toSend, buffer.Bytes()) + logs.Info("header: ", header) + return nil +} + +func (mail SendMail) writeHeader(buffer *bytes.Buffer, Header map[string]string) string { + header := "" + for key, value := range Header { + header += key + ":" + value + "\r\n" + } + header += "\r\n" + buffer.WriteString(header) + return header +} + +// read and write the file to buffer +func (mail SendMail) writeFile(buffer *bytes.Buffer, fileName string) { + file, err := ioutil.ReadFile(fileName) + if err != nil { + panic(err.Error()) + } + payload := make([]byte, base64.StdEncoding.EncodedLen(len(file))) + base64.StdEncoding.Encode(payload, file) + buffer.WriteString("\r\n") + for index, line := 0, len(payload); index < line; index++ { + buffer.WriteByte(payload[index]) + if (index+1)%76 == 0 { + buffer.WriteString("\r\n") + } + } +} + +func SendCommonMail(mailTo, mailCc []string, subject string, body, attchName string) error { + emailName := beego.AppConfig.String("email::email_name") + emailPwd := beego.AppConfig.String("email::email_pwd") + emailHost := beego.AppConfig.String("email::email_host") + emailPwd = "" + emailPort := beego.AppConfig.String("email::email_port") + mailConn := map[string]string{ + "user": emailName, + "pass": emailPwd, + "host": emailHost, + "port": emailPort, + } + port, _ := strconv.Atoi(mailConn["port"]) + m := gomail.NewMessage() + m.SetHeader("From", "cve-manager"+"<"+mailConn["user"]+">") + m.SetHeader("To", mailTo...) + m.SetHeader("Cc", mailCc...) + m.SetHeader("Subject", subject) + m.SetBody("text/plain", body) + //m.Attach(attchName) + _, fileName := filepath.Split(attchName) + m.Attach(attchName, + gomail.Rename(fileName), + gomail.SetHeader(map[string][]string{ + "Content-Disposition": []string{ + fmt.Sprintf(`attachment; filename="%s"`, mime.QEncoding.Encode("UTF-8", fileName)), + }, + }), + ) + d := gomail.NewDialer(mailConn["host"], port, mailConn["user"], mailConn["pass"]) + err := d.DialAndSend(m) + return err +} + +func SendCvrfEmail(subFileSlice []string) { + elt := models.EmailList{EmailType: 1} + el, eltErr := elt.Read("EmailType") + if eltErr != nil { + logs.Error("Failed to get mailing list, err: ", eltErr) + } + toEmailName := []string{} + for _, em := range el { + toEmailName = append(toEmailName, em.EmailName) + } + subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime() + fileStr := strings.Join(subFileSlice, "\r\n") + body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + fileStr) + sendErr := SendCommonMail(toEmailName, []string{}, subject, body, "") + if sendErr == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } +} diff --git a/cve-vulner-manager/taskhandler/xml.go b/cve-vulner-manager/taskhandler/xml.go new file mode 100644 index 0000000000000000000000000000000000000000..b24268bfbd9a268989e3e096a759b225ffa95b78 --- /dev/null +++ b/cve-vulner-manager/taskhandler/xml.go @@ -0,0 +1,437 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "encoding/xml" + "fmt" + "github.com/360EntSecGroup-Skylar/excelize/v2" + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" + "sort" + "strings" + "time" +) + +type Updates struct { + XMLName xml.Name `xml:"updates,omitempty"` + Updatex []Update `xml:"update,omitempty"` +} + +type Update struct { + XMLName xml.Name `xml:"update,omitempty"` + From string `xml:"from,attr"` + Type string `xml:"type,attr"` + Status string `xml:"status,attr"` + Id string `xml:"id"` + Title string `xml:"title"` + Severity string `xml:"severity"` + Release string `xml:"release"` + Issued *Issued `xml:"issued,omitempty"` + References *References `xml:"references,omitempty"` + Description string `xml:"description"` + Pkglist *Pkglist `xml:"pkglist,omitempty"` +} + +type Issued struct { + XMLName xml.Name `xml:"issued,omitempty"` + Date string `xml:"date,attr"` +} + +type References struct { + XMLName xml.Name `xml:"references,omitempty"` + Reference []Reference `xml:"reference,omitempty"` +} + +type Reference struct { + XMLName xml.Name `xml:"reference,omitempty"` + Href string `xml:"href,attr"` + Id string `xml:"id,attr"` + Title string `xml:"title,attr"` + Type string `xml:"type,attr"` +} + +type Pkglist struct { + XMLName xml.Name `xml:"pkglist,omitempty"` + Collection *Collection `xml:"collection,omitempty"` +} + +type Collection struct { + XMLName xml.Name `xml:"collection,omitempty"` + Name string `xml:"name"` + Package []Package `xml:"package,omitempty"` +} + +type Package struct { + XMLName xml.Name `xml:"package,omitempty"` + Arch string `xml:"arch,attr"` + Name string `xml:"name,attr"` + Release string `xml:"release,attr"` + Version string `xml:"version,attr"` + Filename string `xml:"filename"` +} + +// Intermediate information +type SecurityNoticeXml struct { + CveNum string + Title string + Type string + ReferenceLink string + Description string + openEulerScore float64 +} + +type PackRpm struct { + PackName string +} + +type CveXml struct { + PublicDate string + OpenEulerSANum string + Introduction string + CveLevel string + OwnedComponent string + SecurityNotice map[string][]SecurityNoticeXml + PackRpmx map[string][]PackRpm +} + +func ReadXml(filePath string, dpdates *Updates) error { + if filePath == "" || len(filePath) == 0 { + filePath = "download/updateinfo.xml" + } + fisExist, ferr := PathExists(filePath) + if !fisExist { + logs.Error(ferr) + return ferr + } + fd, err := os.Open(filePath) + if err != nil { + logs.Error("open file err : ", err, ",filePath: ", filePath) + return err + } + defer fd.Close() + fileContent, err := ioutil.ReadAll(fd) + if err != nil { + logs.Error("read file err : ", err, ", filePath: ", filePath) + return err + } + err = xml.Unmarshal(fileContent, dpdates) + if err != nil { + logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) + return err + } + return nil +} + +func AbiAffectedVersionBool(abiVersion, branch string) bool { + abiVersionList := []string{} + brandsGroup := strings.Split(abiVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brand = common.BranchVersionRep(brand) + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + prams0 := strings.Replace(brandList[0], " ", "", -1) + if prams == "是" && prams0 == branch { + abiVersionList = append(abiVersionList, brandList[0]) + break + } + } else { + brandList = strings.Split(brand, "：") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + prams0 := strings.Replace(brandList[0], " ", "", -1) + if prams == "是" && prams0 == branch { + abiVersionList = append(abiVersionList, brandList[0]) + break + } + } + } + } + } + if len(abiVersionList) > 0 { + return true + } + return false +} + +func CreateUpdateExcel(excelPath string) (string) { + // File storage directory + sheetName := "cve_list" + xlsx := excelize.NewFile() + index := xlsx.NewSheet(sheetName) + sheetTileMap := make(map[string]string) + sheetTileMap["A1"] = "cve编号" + sheetTileMap["B1"] = "issue编号" + sheetTileMap["C1"] = "issue所属仓库" + sheetTileMap["D1"] = "score" + sheetTileMap["E1"] = "version" + sheetTileMap["F1"] = "abi是否变化" + for k, v := range sheetTileMap { + xlsx.SetCellValue(sheetName, k, v) + } + xlsx.SetActiveSheet(index) + err := xlsx.SaveAs(excelPath) + if err != nil { + logs.Error(err) + return "" + } + return excelPath +} + +func CreateUpdateData(its models.IssueTemplate, affectBranch string) []interface{} { + ownedVersion := "" + cveData := make([]interface{}, 0) + cveData = append(cveData, its.CveNum) + cveData = append(cveData, its.IssueNum) + cveData = append(cveData, its.OwnedComponent) + cveData = append(cveData, its.OpenEulerScore) + ownedVersion = its.OwnedVersion + ori := models.QueryEulerOriginBRepo(its.OwnedComponent, affectBranch) + if len(ori) > 0 { + for _, or := range ori { + if len(or.Branchs) > 0 { + brSlice := strings.Split(or.Branchs, ",") + if len(brSlice) > 0 { + for _, br := range brSlice { + if br == affectBranch { + if len(or.Version) > 0 && len(or.Version) < 32 { + ownedVersion = or.Version + goto toVersion + } + } + } + } + } + } + } +toVersion: + cveData = append(cveData, ownedVersion) + abiBool := AbiAffectedVersionBool(its.AbiVersion, affectBranch) + if abiBool { + cveData = append(cveData, "是") + } else { + cveData = append(cveData, "否") + } + return cveData +} + +func ReadWriteUpdateExcel(excelPath, ownedComponent, cveNum, affectBranch string) error { + sheetName := "cve_list" + file, openErr := excelize.OpenFile(excelPath) + if openErr != nil { + logs.Error("fail to open the file, excelPath: ", excelPath, ", openErr: ", openErr) + return openErr + } + vc := models.QueryCveByPackName(cveNum, ownedComponent, 1) + if len(vc) > 0 { + for _, v := range vc { + issueTmp := models.IssueTemplate{CveId: v.CveId, Status: 3} + err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "Status") + if err == nil { + cveData := CreateUpdateData(issueTmp, affectBranch) + if len(cveData) > 0 { + rows, sheetErr := file.GetRows(sheetName) + if sheetErr != nil { + logs.Error(sheetErr) + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + setErr := file.SetSheetRow(sheetName, axis, &cveData) + if setErr != nil { + logs.Error("setErr: ", setErr) + } + } + } + } + } + fileErr := file.SaveAs(excelPath) + if fileErr != nil { + logs.Error("Failed to save file, ", fileErr) + } + return fileErr +} + +func WriteXml(filePath, excelName, affectBranch string, cveXmlList []CveXml, dpdates *Updates, + securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { + for _, cveXml := range cveXmlList { + upDatex := Update{From: "openeuler.org", Type: "security", Status: "stable", Id: cveXml.OpenEulerSANum, + Title: cveXml.Introduction, Severity: cveXml.CveLevel, Release: "openEuler"} + var pubDate Issued + pubDate.Date = cveXml.PublicDate + upDatex.Issued = &pubDate + openEulerSANum := "" + var rfs References + if sn, ok := securityNotice[cveXml.OwnedComponent]; ok { + if len(sn) > 0 { + descriptionsList := make([]string, 0) + openEulerScoreSlice := make([]float64, 0) + for _, sec := range sn { + var rf Reference + rf.Id = sec.CveNum + rf.Title = sec.CveNum + rf.Href = sec.ReferenceLink + rf.Type = "cve" + descriptionsList = append(descriptionsList, sec.Description) + openEulerScoreSlice = append(openEulerScoreSlice, sec.openEulerScore) + rfs.Reference = append(rfs.Reference, rf) + if len(openEulerSANum) < 2 { + cfc := models.GetCvrfSaRecordByCve(sec.CveNum) + if len(cfc) > 0 { + for _, cf := range cfc { + if len(cf.SortOpenEulerSANum) > 1 { + openEulerSANum = cf.SortOpenEulerSANum + break + } + } + } + } + // write excel + ReadWriteUpdateExcel(excelName, cveXml.OwnedComponent, sec.CveNum, affectBranch) + } + upDatex.Description = strings.Join(descriptionsList, " ") + if len(openEulerScoreSlice) > 1 { + sort.Float64s(openEulerScoreSlice) + } + cveLevel := models.OpenEulerScoreProc(openEulerScoreSlice[len(openEulerScoreSlice)-1]) + if strings.ToLower(cveLevel) == "low" { + upDatex.Severity = "Low" + } else if strings.ToLower(cveLevel) == "medium" { + upDatex.Severity = "Moderate" + } else if strings.ToLower(cveLevel) == "high" { + upDatex.Severity = "Important" + } else { + upDatex.Severity = "Critical" + } + } + } + if len(openEulerSANum) > 2 { + upDatex.Id = openEulerSANum + } + upDatex.References = &rfs + var pl Pkglist + var ct Collection + ct.Name = "openEuler" + if prx, ok := packRpmx[cveXml.OwnedComponent]; ok { + if len(prx) > 0 { + for _, pr := range prx { + if strings.Contains(pr.PackName, ".src.") { + continue + } + var pe Package + pe.Filename = pr.PackName + //pe.Name = cveXml.OwnedComponent + if len(pr.PackName) > len(cveXml.OwnedComponent) { + //packVersion := pr.PackName[len(cveXml.OwnedComponent) + 1: len(pr.PackName) - 4] + packVersionList := strings.Split(pr.PackName, "-") + if len(packVersionList) >= 3 { + pe.Version = packVersionList[len(packVersionList)-2] + rpmName := packVersionList[len(packVersionList)-1][:len(packVersionList[len(packVersionList)-1])-4] + lastIndex := strings.LastIndexAny(rpmName, ".") + if lastIndex != -1 { + pe.Release = rpmName[:lastIndex] + pe.Arch = rpmName[lastIndex+1:] + } + pe.Name = strings.Join(packVersionList[0:len(packVersionList)-2], "-") + } + } + ct.Package = append(ct.Package, pe) + } + } + } + pl.Collection = &ct + upDatex.Pkglist = &pl + dpdates.Updatex = append(dpdates.Updatex, upDatex) + } + xmlOutPut, outPutErr := xml.MarshalIndent(dpdates, "", " ") + if outPutErr == nil { + headerBytes := []byte(xml.Header) + xmlOutPutData := append(headerBytes, xmlOutPut...) + os.Remove(filePath) + ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) + } else { + logs.Error(outPutErr) + } +} + +func BuildXml(cveXml *[]CveXml, v *models.ExcelExport, + securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm, affectBranch string) { + pkg, pkgErr := models.GetCvePackageList(v.SecID, affectBranch) + if pkgErr != nil { + logs.Error(pkgErr) + return + } + //cvex.SecurityNotice = make(map[string][]SecurityNoticeXml) + sn := SecurityNoticeXml{CveNum: v.CveNum, Title: v.CveNum, Type: "cve", ReferenceLink: v.ReferenceLink, + Description: deleteTailBlank(v.CveBrief) + "(" + v.CveNum + ")", openEulerScore: v.OpenEulerScore} + if vx, ok := securityNotice[v.OwnedComponent]; !ok { + snArry := []SecurityNoticeXml{} + snArry = append(snArry, sn) + securityNotice[v.OwnedComponent] = snArry + var cvex CveXml + if v.PublicDate == "" { + v.PublicDate = time.Now().Format("2006-01-02") + } + cvex.OpenEulerSANum = v.OpenEulerSANum + cvex.PublicDate = v.PublicDate + if strings.ToLower(v.CveLevel) == "low" { + cvex.CveLevel = "Low" + } else if strings.ToLower(v.CveLevel) == "medium" { + cvex.CveLevel = "Moderate" + } else if strings.ToLower(v.CveLevel) == "high" { + cvex.CveLevel = "Important" + } else { + cvex.CveLevel = "Critical" + } + if len(v.Introduction) > 1 { + cvex.Introduction = v.Introduction[:len(v.Introduction)-1] + } else { + cvex.Introduction = v.Introduction + } + cvex.OwnedComponent = v.OwnedComponent + *cveXml = append(*cveXml, cvex) + } else { + sameFlag := false + for _, vu := range vx { + if sn.CveNum == vu.CveNum { + sameFlag = true + break + } + } + if !sameFlag { + vx = append(vx, sn) + securityNotice[v.OwnedComponent] = vx + } + } + //cvex.PackRpmx = make(map[string][]PackRpm) + if px, ok := packRpmx[v.OwnedComponent]; !ok { + prArry := []PackRpm{} + for _, v := range pkg { + pr := PackRpm{PackName: v.PackName} + prArry = append(prArry, pr) + } + packRpmx[v.OwnedComponent] = prArry + } else { + for _, v := range pkg { + pnFlag := false + for _, p := range px { + if p.PackName == v.PackName { + pnFlag = true + break + } + } + if !pnFlag { + pr := PackRpm{PackName: v.PackName} + px = append(px, pr) + } + } + packRpmx[v.OwnedComponent] = px + } + return +} diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go new file mode 100644 index 0000000000000000000000000000000000000000..f5b73fc10e62fee45161f8963ede8326faf0b558 --- /dev/null +++ b/cve-vulner-manager/taskhandler/yaml.go @@ -0,0 +1,656 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "encoding/json" + "errors" + "fmt" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "net/http" + "net/url" + "strconv" + "strings" + "sync" +) + +var lock sync.Mutex + +func GetYamlTables(url string) (string, error) { + compURL := url + "/lifeCycle/tables" + body, err := util.HTTPGetCom(compURL) + if err == nil && body != nil { + var respBody map[string]interface{} + err = json.Unmarshal(body, &respBody) + if err != nil { + logs.Error(err) + return "", err + } + logs.Info(respBody) + if respBody["code"].(string) == "2001" { + for _, values := range respBody["data"].([]interface{}) { + var gt models.GitPackageTable + gt.TableName = values.(string) + tableID, err := models.CreateYamlTable(>) + if tableID > 0 { + logs.Info("table: ", values, "Inserted successfully, table_id: ", tableID) + } else { + logs.Error("table: ", values, "Insertion failed, err: ", err) + return "", err + } + } + } else { + return "", errors.New("数据格式错误") + } + } + return "", nil +} + +//GetYamlByGit +func GetYamlByGit(url string) (string, error) { + defer common.Catchs() + var gt []models.GitPackageTable + page := 1 + size := 20 + num, err := models.GetYamlTable(>) + if err != nil { + logs.Error("Query table failed, unable to get yaml, err: ", err) + return "", err + } + logs.Info("There are: ", num, ", Table data needs to be obtained, ", gt) + compURL1 := url + "/packages" + var ch = make(chan int, len(gt)) + for i, tableValue := range gt { + go func(tv models.GitPackageTable, idx, locPage, locSize int, locUrl, locCompURL1 string) { + ok, err := GetYaml(locUrl, locCompURL1, locPage, locSize, tv, &ch) + if err == nil { + logs.Info("The current data processing is successful,i: ", idx) + } else { + logs.Error("The current data processing failed, ok: ", ok, ",i: ", idx, ", err: ", err) + } + }(tableValue, i, page, size, url, compURL1) + } + for i := 0; i < len(gt); i++ { + <-ch + } + close(ch) + return "", nil +} + +func GetYaml(url, compURL1 string, page, size int, + tableValue models.GitPackageTable, ch *chan int) (string, error) { + defer common.Catchs() + var tc GitTablePackCount + tc.TableName = tableValue.TableName + tc.Page = 0 + tc.Size = 0 + tc.Page = page + tc.Size = size + compURL2 := compURL1 + "?table_name=" + tableValue.TableName + for ; ; { + compURL := compURL2 + + "&page_num=" + strconv.Itoa(tc.Page) + "&page_size=" + strconv.Itoa(size) + body, err := util.HTTPGetCom(compURL) + var respBody map[string]interface{} + if err == nil && body != nil { + err = json.Unmarshal(body, &respBody) + if err != nil { + logs.Error(err) + return "", err + } + } else { + logs.Error("http request failed, url: ", compURL) + return "", err + } + logs.Info("start: ", tc) + if respBody["code"].(string) == "2001" { + curCount := 0 + if respBody == nil || respBody["data"] == nil || len(respBody["data"].([]interface{})) == 0 { + logs.Error("Data is empty, url: ", compURL) + return "", err + } + for i, values := range respBody["data"].([]interface{}) { + tc.Count = tc.Count + 1 + if values == nil || values == "" || len(values.(map[string]interface{})) == 0 { + curCount = curCount + 1 + continue + } + valuesX := values.(map[string]interface{}) + lock.Lock() + ok, err := ProcPackDetail(url, valuesX, tableValue, i) + lock.Unlock() + logs.Info("ok: ", ok, ", err: ", err) + curCount = curCount + 1 + } + totalPage := 0 + switch respBody["total_page"].(type) { + case string: + totalPage, _ = strconv.Atoi(respBody["total_page"].(string)) + case int: + totalPage = respBody["total_page"].(int) + case int64: + totalPage = int(respBody["total_page"].(int64)) + case float64: + totalPage = int(int64(respBody["total_page"].(float64))) + default: + totalPage = 1 + } + totalCount := 0 + switch respBody["total_count"].(type) { + case string: + totalCount, _ = strconv.Atoi(respBody["total_count"].(string)) + case int: + totalCount = respBody["total_count"].(int) + case int64: + totalCount = int(respBody["total_count"].(int64)) + case float64: + totalCount = int(int64(respBody["total_count"].(float64))) + default: + totalCount = 1 + } + if tc.Page > totalPage || tc.Size >= totalCount { + logs.Info("Processed：tableName: ", tc.TableName, "Data acquisition completed， "+ + "total pages(page_num)：", totalPage, ", Total number(page_size)：", totalCount, "\n", + ",current page number：", tc.Page, ",Current number: ", tc.Size, ",url: ", compURL) + break + } else { + logs.Info("current： tableName: ", tc.TableName, "Data acquisition completed， "+ + "total pages(page_num)：", totalPage, ", Total number(page_size)：", totalCount, "\n", + ",current page number：", tc.Page, ",Current number: ", tc.Size, ",url: ", compURL) + lock.Lock() + tc.Page = tc.Page + page + tc.Size = tc.Size + curCount + lock.Unlock() + logs.Info("After the increase： tableName: ", tc.TableName, "Data acquisition completed， "+ + "total pages(page_num)：", totalPage, ", Total number(page_size)：", totalCount, "\n", + ",current page number：", tc.Page, ",Current number: ", tc.Size, ",url: ", compURL) + } + logs.Info("start: ", tc) + } else { + logs.Error("Network request failed，url:", compURL) + continue + } + } + *ch <- 1 + return "", nil +} + +func ProcPackDetail(url string, value map[string]interface{}, tableValue models.GitPackageTable, i int) (string, error) { + values := value + var ge models.GitOpenEuler + GitOpenEulerData(values, &ge, tableValue) + // Query cpe data + var pc models.PackageCpe + pcError := models.GetCpePackName(&pc, ge.PackageName) + if pcError == nil && pc.Id > 0 { + ge.CpePackName = pc.CpePackName + } + logs.Info("Being processed packageName: ", ge.PackageName) + ok, _ := models.GetSingleYaml(&ge) + if ok { + logs.Info("PackageName: ", ge.PackageName, ",Already exists, no need to insert again") + gpinfo, okinfo := models.QueryCveOpeneulerDetaildataByName(ge.PackageName, ge.Version) + if !okinfo && gpinfo.DetailId == 0 { + _, pierr := GetPackageInfo(url, tableValue, ge) + if pierr != nil { + logs.Error("Failed to get package details, url: ", url, + ", PackageName: ", ge.PackageName, ",version: ", ge.Version, ",err: ", pierr) + return ge.PackageName, pierr + } + } + return ge.PackageName, nil + } + gitID, typeX, err := models.CreateYaml(&ge, tableValue) + if gitID > 0 && err == nil { + logs.Info("PackageName: ", ge.PackageName, typeX, "success, git_id: ", gitID) + } else { + logs.Error("PackageName: ", ge.PackageName, typeX, "failure, err: ", err) + if strings.ContainsAny(string(err.Error()), "Duplicate entry") { + ok, _ := models.GetSingleYaml(&ge) + if ok { + logs.Info("PackageName: ", ge.PackageName, ",Already exists, no need to insert again") + gpinfo, okinfo := models.QueryCveOpeneulerDetaildataByName(ge.PackageName, ge.Version) + if !okinfo && gpinfo.DetailId == 0 { + _, pierr := GetPackageInfo(url, tableValue, ge) + if pierr != nil { + logs.Error("Failed to get package details, url: ", url, ", PackageName: ", ge.PackageName, + ",version: ", ge.Version, ",err: ", pierr) + return ge.PackageName, pierr + } + } + return ge.PackageName, nil + } + } + return "", err + } + if typeX == "insert" && gitID > 0 { + _, pierr := GetPackageInfo(url, tableValue, ge) + if pierr != nil { + logs.Error("Failed to get package details, url: ", url, ", PackageName: ", ge.PackageName, ",version: ", ge.Version, ",err: ", pierr) + return ge.PackageName, pierr + } + } + return "", nil +} + +func GetPackageInfo(urls string, tableValue models.GitPackageTable, ge models.GitOpenEuler) (string, error) { + scpURL := urls + "/packages/packageInfo" + "?table_name=" + tableValue.TableName + "&pkg_name=" + ge.PackageName + logs.Info("PackageName: ", ge.PackageName, "Get packageInfo, request parameters: ", scpURL) + var urlS url.URL + q := urlS.Query() + q.Add("table_name", tableValue.TableName) + q.Add("pkg_name", ge.PackageName) + params := q.Encode() + req, _ := http.NewRequest("GET", urls, nil) + req.URL = &url.URL{ + Scheme: req.URL.Scheme, + Host: req.URL.Host, + RawQuery: params, + Path: "/pkgmanage/packages/packageInfo", + } + + req.URL.EscapedPath() + body, err := util.HTTPGetCom(req.URL.String()) + var respBody map[string]interface{} + if err == nil && body != nil { + err = json.Unmarshal(body, &respBody) + if err != nil { + logs.Error(err) + return "", err + } + } else { + logs.Error("http request failed, scpURL: ", scpURL) + return "", err + } + logs.Info(respBody) + if respBody["code"].(string) == "2001" { + if respBody["data"] == nil || respBody["data"] == "" { + logs.Error("Failed to get package details, scpURL: ", scpURL) + return "", errors.New("数据错误") + } + var gp models.GitPackageInfo + GitOpenEulerInfoData(respBody["data"].(map[string]interface{}), &gp, ge) + detailid, typex, err := models.CreateYamlDetail(&gp, ge) + if detailid > 0 && typex == "insert" && err == nil { + subdata := respBody["data"].(map[string]interface{}) + if subdata["subpack"] == nil { + return "", err + } + if subdata == nil || subdata["subpack"] == nil || len(subdata["subpack"].([]interface{})) == 0 { + return "", errors.New("数据错误") + } + subpack := subdata["subpack"].([]interface{}) + for _, packValuex := range subpack { + logs.Info("yaml packValuex: ", packValuex) + if packValuex == nil || packValuex == "" || len(packValuex.(map[string]interface{})) == 0 { + continue + } + packValue := packValuex.(map[string]interface{}) + var gb models.GitSubPack + gb.DetailId = gp.DetailId + switch packValue["id"].(type) { + case string: + gb.Ids, _ = strconv.ParseInt(packValue["id"].(string), 10, 64) + case int: + gb.Ids = packValue["id"].(int64) + case int64: + gb.Ids = packValue["id"].(int64) + case float64: + gb.Ids = int64(packValue["id"].(float64)) + default: + gb.Ids = 0 + } + if packValue["name"] == nil { + gb.SubPackName = "" + } else { + gb.SubPackName = packValue["name"].(string) + } + SubID, typex, err := models.CreateYamlSubPack(&gb) + if SubID > 0 && typex == "insert" && err == nil { + if packValue["provides"] != nil && len(packValue["provides"].([]interface{})) > 0 { + provides := packValue["provides"].([]interface{}) + for _, provValuex := range provides { + logs.Info("yaml provValuex: ", provValuex) + if provValuex == nil || provValuex == "" || len(provValuex.(map[string]interface{})) == 0 { + continue + } + provValue := provValuex.(map[string]interface{}) + var gs models.GitSubPackProvides + gs.SubId = SubID + switch provValue["id"].(type) { + case string: + gs.Ids, _ = strconv.ParseInt(provValue["id"].(string), 10, 64) + case int: + gs.Ids = provValue["id"].(int64) + case int64: + gs.Ids = provValue["id"].(int64) + case float64: + gs.Ids = int64(provValue["id"].(float64)) + default: + gs.Ids = 0 + } + if provValue["name"] == nil { + gs.ProvideName = "" + } else { + gs.ProvideName = provValue["name"].(string) + } + gs.Requiredby = "" + ProvideID, typexx, err := models.CreateYamlSubPackProvides(&gs) + if ProvideID > 0 && typexx == "insert" && err == nil { + if provValue["requiredby"] != nil && len(provValue["requiredby"].([]interface{})) > 0 { + requiredby := provValue["requiredby"].([]interface{}) + for _, reqValue := range requiredby { + if reqValue != nil && reqValue.(string) != "" { + var gr models.GitSubPackRequiredby + gr.ProvideId = gs.ProvideId + gr.Requiredby = reqValue.(string) + ID, typexy, err := models.CreateYamlSubPackRequiredb(&gr) + logs.Info("CreateYamlSubPackRequiredb", ID, typexy, err) + } + } + } + } + } + } + if packValue["requires"] != nil && len(packValue["requires"].([]interface{})) > 0 { + requires := packValue["requires"].([]interface{}) + for _, reqValuexx := range requires { + logs.Info("reqValuexx: ", reqValuexx) + if reqValuexx == nil || reqValuexx == "" || len(reqValuexx.(map[string]interface{})) == 0 { + continue + } + reqValuex := reqValuexx.(map[string]interface{}) + reqStr := "" + if reqValuex["providedby"] != nil && len(reqValuex["providedby"].([]interface{})) > 0 { + providedby := reqValuex["providedby"].([]interface{}) + for _, reqValue := range providedby { + if reqValue != nil && reqValue.(string) != "" { + reqStr = reqStr + reqValue.(string) + "," + } + } + } + if reqStr != "" { + reqStr = reqStr[:len(reqStr)-1] + } + var gs models.GitSubPackRequire + gs.SubId = SubID + switch reqValuex["id"].(type) { + case string: + gs.Ids, _ = strconv.ParseInt(reqValuex["id"].(string), 10, 64) + case int: + gs.Ids = reqValuex["id"].(int64) + case int64: + gs.Ids = reqValuex["id"].(int64) + case float64: + gs.Ids = int64(reqValuex["id"].(float64)) + default: + gs.Ids = 0 + } + if reqValuex["name"] == nil { + gs.RequireName = "" + } else { + gs.RequireName = reqValuex["name"].(string) + } + gs.Providedby = reqStr + RequireID, typexx, err := models.CreateYamlSubPackRequires(&gs) + logs.Info("CreateYamlSubPackRequires", RequireID, typexx, err) + } + } + } + } + } + } + return "", nil +} + +func GetEulerYamlInfo(url string) (string, error) { + defer common.Catchs() + compUrl := url + "?community=openeuler" + body, err := util.HTTPGetCom(compUrl) + var respBody map[string]interface{} + if err == nil && body != nil { + err = json.Unmarshal(body, &respBody) + if err != nil { + logs.Error(err) + return "", err + } + } else { + logs.Error("http Request failed， url: ", compUrl) + return "", err + } + if respBody == nil || respBody["data"] == nil || len(respBody["data"].([]interface{})) == 0 { + logs.Error("Data is empty, url: ", compUrl) + return "", fmt.Errorf("Data is empty, url: %s", compUrl) + } + if respBody["code"].(float64) == 200 && respBody["total"].(float64) > 0 { + //models.TruncateOpenEulerRepoOrigin() + outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch") + outSyncBranchList := strings.Split(outSyncBranch, ",") + for _, values := range respBody["data"].([]interface{}) { + valuesX := values.(map[string]interface{}) + ProcYamlInfo(valuesX, outSyncBranchList) + } + } + return "", nil +} + +// Parse the received json data +func ProcYamlInfo(valuesX map[string]interface{}, outSyncBranch []string) { + if valuesX != nil && len(valuesX) > 0 { + for repKey, repValue := range valuesX { + if repKey != "" && len(repKey) > 1 { + //Delete all data first, then insert new data + //models.DelOpenEulerRepoOriginByName(repKey) + value := repValue.(map[string]interface{}) + brValue, brOk := value["branches"] + if brOk { + brKey := brValue.([]interface{}) + if brKey != nil && len(brKey) > 0 { + for _, branchValue := range brKey { + var er models.OpenEulerRepoOrigin + er.PackageName = repKey + v := branchValue.(map[string]interface{}) + for vKey, vu := range v { + switch vKey { + case "summary": + er.Summary = vu.(string) + case "version": + er.Version = vu.(string) + case "brname": + er.Branchs = vu.(string) + case "description": + er.Decription = vu.(string) + } + } + if len(er.Branchs) > 0 { + outFlag := false + for _, outBranch := range outSyncBranch { + if outBranch == er.Branchs { + outFlag = true + break + } + } + if outFlag { + continue + } + } + UpdateYamlToDb(er) + } + } + } + } + } + } + return +} + +// Update data to the database +func UpdateYamlToDb(er models.OpenEulerRepoOrigin) { + if len(er.PackageName) > 0 && len(er.Version) > 0 && !strings.Contains(er.Version, "%") { + var oro models.OpenEulerRepoOrigin + oro.PackageName = er.PackageName + oro.Version = er.Version + repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version") + if oro.Id > 0 { + if len(oro.Decription) < 2 && len(er.Decription) > 2 { + oro.Decription = er.Decription + } + if len(er.Branchs) > 1 && len(oro.Branchs) > 1 { + BranchSlice := strings.Split(oro.Branchs, ",") + if len(BranchSlice) > 0 { + branchFlag := false + for _, br := range BranchSlice { + if br == er.Branchs { + branchFlag = true + break + } + } + if !branchFlag { + oro.Branchs = oro.Branchs + "," + er.Branchs + } + } + } else { + if len(er.Branchs) > 1 && len(oro.Branchs) < 2 { + oro.Branchs = er.Branchs + } + } + if len(oro.Summary) < 2 && len(er.Summary) > 2 { + oro.Summary = er.Summary + } + oro.UpdateTime = er.UpdateTime + oro.Status = 1 + oro.Repo = er.PackageName + upErr := models.UpdateEulerRepoOrigin(&oro, "Decription", "Branchs", "Summary", "UpdateTime", "Repo") + logs.Info("UpdateEulerRepoOrigin, upErr: ", upErr) + } else { + logs.Info(repErr) + oro.Decription = er.Decription + oro.Branchs = er.Branchs + oro.Summary = er.Summary + oro.UpdateTime = er.UpdateTime + oro.PackageName = er.PackageName + oro.Version = er.Version + oro.Status = 1 + oro.Repo = er.PackageName + oro.CreateTime = common.GetCurTime() + id, inErr := models.InsertEulerRepoOrigin(&oro) + logs.Info("InsertEulerRepoOrigin, id, inErr: ", id, inErr) + } + } else { + logs.Error("UpdateYamlToDb, The version information of the current warehouse is wrong, er: ", er) + } +} + +// Synchronize other sources of yaml version +func SyncEulerYaml() { + // Query raw data + yamlId := int64(0) + for { + yamlList := models.GetEulerYamlList(yamlId, 100) + if len(yamlList) > 0 { + for _, yl := range yamlList { + if len(yl.PackageName) < 2 || len(yl.Version) < 1 { + logs.Error("yl: ", yl) + continue + } + yamlId = yl.Id + var ge models.GitOpenEuler + ge.PackageName = yl.PackageName + ge.Version = yl.Version + geErr := models.QueryYamlData(&ge, "PackageName", "Version") + if ge.GitId == 0 { + logs.Info("yaml data does not exist, need to insert data, err: ", + geErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version) + AddYamlData(&ge, yl) + } else { + EditYamlData(&ge, yl) + } + } + } else { + break + } + } +} + +// Delete historical yaml source version data that does not exist +func DelHistoryEulerYaml() { + gitId := int64(0) + for { + yamlList := models.GetGitOpenEulerList(gitId, 100) + if len(yamlList) > 0 { + for _, yl := range yamlList { + if len(yl.PackageName) < 2 || len(yl.Version) < 1 { + logs.Error("yl: ", yl) + continue + } + gitId = yl.GitId + var oro models.OpenEulerRepoOrigin + oro.PackageName = yl.PackageName + oro.Version = yl.Version + repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version") + if oro.Id == 0 { + logs.Info("yaml data does not exist, delete data, err: ", + repErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version) + yl.Status = 2 + upErr := models.UpdateYamlData(&yl, "Status") + if upErr == nil { + ge := models.GitPackageInfo{PackageName: yl.PackageName, Version: yl.Version} + qErr := models.QueryGitPackageInfo(&ge, "PackageName", "Version") + if qErr == nil { + ge.Status = 2 + upgErr := models.UpdateGitPackageInfo(&ge, "Status") + if upgErr != nil { + logs.Error("UpdateGitPackageInfo, upgErr: ", upgErr) + } + } + } + } + } + } else { + break + } + } +} + +func AddYamlData(ge *models.GitOpenEuler, yl models.OpenEulerRepoOrigin) { + ge.PackageName = yl.PackageName + ge.Version = yl.Version + ge.PackageId = 0 + ge.TableId = 0 + //ge.TableName = yl.Branchs + ge.Status = 1 + ymalId, yErr := models.InsertYamlData(ge) + if yErr == nil { + ProcGitPackageInfo(ymalId, yl) + } +} + +func EditYamlData(ge *models.GitOpenEuler, yl models.OpenEulerRepoOrigin) { + ge.Status = 1 + yErr := models.UpdateYamlData(ge, "Status") + logs.Info("UpdateYamlData, yErr: ", yErr) + ProcGitPackageInfo(ge.GitId, yl) +} + +func ProcGitPackageInfo(ymalId int64, yl models.OpenEulerRepoOrigin) { + gp := models.GitPackageInfo{GitId: ymalId, PackageName: yl.PackageName, Version: yl.Version} + gpErr := models.QueryGitPackageInfo(&gp, "GitId", "PackageName", "Version") + if gpErr != nil { + gp.GitId = ymalId + gp.PackageName = yl.PackageName + gp.Version = yl.Version + gp.Summary = yl.Summary + gp.Decription = yl.Decription + gp.Status = 0 + id, iErr := models.InsertGitPackageInfo(&gp) + logs.Info("InsertGitPackageInfo, id, iErr", id, iErr) + } else { + gp.Summary = yl.Summary + gp.Decription = yl.Decription + gp.Status = 0 + uErr := models.UpdateGitPackageInfo(&gp, "Summary", "Decription", "Status") + logs.Info("UpdateGitPackageInfo, uErr: ", uErr) + } +} diff --git a/cve-vulner-manager/tests/aes_test.go b/cve-vulner-manager/tests/aes_test.go new file mode 100644 index 0000000000000000000000000000000000000000..aa06bb9917a37a3dbc81c8e003196d9a3fc90a71 --- /dev/null +++ b/cve-vulner-manager/tests/aes_test.go @@ -0,0 +1,18 @@ +package test + +import ( + "cvevulner/common" + "github.com/astaxie/beego" + "testing" +) + +func TestEnPwdCode(t *testing.T) { + pwd := []byte("159xwz@Qmxx") + key := []byte(beego.AppConfig.String("key")) + code, err := common.EnPwdCode(pwd, key) + if err != nil { + t.Error(err) + }else { + t.Log(code) + } +} diff --git a/cve-vulner-manager/tests/conf/app.conf b/cve-vulner-manager/tests/conf/app.conf new file mode 100644 index 0000000000000000000000000000000000000000..4020e793ba2a7c7e0a7aef7a400d0fd757a2fc76 --- /dev/null +++ b/cve-vulner-manager/tests/conf/app.conf @@ -0,0 +1,107 @@ +appname = cvevulner +#appname = cve_manager +httpport = 80 +runmode = dev +autorender = false +copyrequestbody = true +EnableDocs = true +sqlconn = +# 登录双方约定的key +key = "${AES_KEY||djS*@+8K9{J!ymk6}" +initdb = 1 +# 登录key +loginkey = "{${LOGIN_KEY||djS*@+8K9{-!yo%64}" +# token 有效期,单位：天 +token_expir_time=3 +#分析指令 +analysisCmd = "/analysis" +endCmd = "/done" +rejectCmd = "/reject" +approveCmd = "/approve" + + +[mysql] +#dbhost = 127.0.0.1 +dbhost = 159.138.2.2 +dbport = 3306 +#dbuser = "${DB_USER||root}" +dbuser = "${DB_USER||cve}" +#dbpwd = "${DB_PWD||***}" +dbpwd = "+KAOKtEnEn7gRlDEIjeh59lutAMw2688YOqKVJQ2+Lo=" +dbname = cvevulner +dbprefix = cve_ +maxidle = 30 +maxconn = 3000 + + +[log] +log_level = 7 +#log_path = C:\GoPject\src\cvevulner\logs\cve.log +log_path = logs/cve.log +maxlines=200000 +maxsize=1024000 + +[crontab] +ymalflag = 2 +getymal = 00 00 23 * * * +<<<<<<< HEAD +cveflag = 2 +======= +cveflag = 1 +>>>>>>> dcc533bfb3e46ca7de687eb69f20e183f2353c67 +getcve = 00 00 01 * * * +oricveflag = 2 +oricvecheck = 00 00 02 * * * +getissueflag = 2 +getissue = 00 00 03 * * * +issueflag = 2 +createissue = * * 01 * * * +test = 0/10 * * * * * +gittokenflag = 1 +issueoath = * * */20 * * * +genexcelflag = 2 +genexcel = 00 00 04 * * * +days = -30 +prcnum = 100 + + +[gitee] +#owner = xwzQmxx +#path = test +#email = 1499273991@qq.com +#redirect_uri = http://119.8.126.102:80/v1/issue/oauth/callback +# -------jianjun gitee 配置 -------- +owner = zhangjianjun_code +path = jasper +email = 7844966+zhangjianjun_code@user.noreply.gitee.com +redirect_uri = http://159.138.2.2:80/v1/issue/oauth/callback + +scope = user_info projects pull_requests issues notes keys hook groups gists enterprises emails +# 优先从系统环境变量获取 获取失败使用默认值 **** +client_id = "${GITEE_CLIENT_ID||****}" +client_secret = "${GITEE_CLIENT_SECRET||****}" +password = "${GITEE_PASSWORD||****}" + +# git token +git_token = "${GITEE_TOKEN||xxx}" + +[hook] +hookpwd = "${HOOK_PWD||***}" +hookurl = http://159.138.2.2:80/v1/issue/hook/event +#hookurl = http://119.8.126.102:80/v1/issue/hook/event + + +[yaml] +apiurl = https://api.openeuler.org/pkgmanage + +[cve] +cveref = https://nvd.nist.gov/vuln/detail/ +openeulernum = 3000 + +[reflink] +comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md + +[excel] +forcerewrite = false +snprefix = op-2020-10- +snsuffix = 1002 \ No newline at end of file diff --git a/cve-vulner-manager/tests/createrepo.go b/cve-vulner-manager/tests/createrepo.go new file mode 100644 index 0000000000000000000000000000000000000000..01ff0d44242060083e4fe12e43958b80a34c9d37 --- /dev/null +++ b/cve-vulner-manager/tests/createrepo.go @@ -0,0 +1,97 @@ +package test + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "os" +) + +func QueryOriginCveExcelData() ([]models.OriginExcel, int64, error) { + o := orm.NewOrm() + var coe []models.OriginExcel + num, err := o.Raw("select cve_id,cve_num,cve_url, cve_version, pack_name," + + "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity,"+ + "access_complexity,privilege_required,user_interaction,scope,confidentiality,integrity,availability,authentication," + + "cve_status,create_time from cve_origin_excel GROUP BY pack_name").QueryRows(&coe) + if err == nil && num > 0 { + logs.Info("cve_origin_excel 查询结果： ", num) + } else { + logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) + } + return coe, num, err +} + +func UpdateExcelDataPkName(originPackName, packName string) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_origin_excel SET pack_name = ?" + + " WHERE pack_name = ?", packName, originPackName).QueryRow() + return +} + +func CreateRepo() { + coe, num, err := QueryOriginCveExcelData() + if err == nil && num > 0 { + for _, qc := range coe { + PostRepo(qc.PackName) + } + } +} + +func PostRepo(packName string) error{ + url := "https://gitee.com/api/v5/user/repos" + accessToken := os.Getenv("GITEE_TOKEN") + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "name": "%s", + "has_issues": "true", + "has_wiki": "true", + "can_comment": "true", + "auto_init": "true", + "path": "%s", + "private": "false" + }`, accessToken, packName, packName) + logs.Info("isssue_body: ", requestBody) + resp, err := util.HttpPost(url, requestBody) + if err != nil { + logs.Error("创建",packName,"失败, err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建issue 失败, err: ", ok, "url: ", url) + return errors.New("创建仓库失败") + } + errb := CreateBrand(packName) + logs.Info(errb) + return nil +} + +func CreateBrand(packName string) error{ + url := "https://gitee.com/api/v5/repos/zhangjianjun_code/" + packName + "/branches" + accessToken := os.Getenv("GITEE_TOKEN") + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "refs": "master", + "branch_name": "openEuler-20.03-LTS" + }`, accessToken) + logs.Info("isssue_body: ", requestBody) + resp, err := util.HttpPost(url, requestBody) + if err != nil { + logs.Error("创建分支：openEuler-20.03-LTS",packName,"失败, err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建分支：openEuler-20.03-LTS 失败, err: ", ok, "url: ", url) + return errors.New("创建分支失败") + } + + return nil +} + +func Init() { + CreateRepo() +} \ No newline at end of file diff --git a/cve-vulner-manager/tests/createrepo_test.go b/cve-vulner-manager/tests/createrepo_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3750e661b5ac7ca605af6cc22c5a6425fd45df94 --- /dev/null +++ b/cve-vulner-manager/tests/createrepo_test.go @@ -0,0 +1,144 @@ +package test + +import ( + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "os" +) + +func QueryOriginCveExcelData() ([]models.OriginExcel, int64, error) { + o := orm.NewOrm() + var coe []models.OriginExcel + num, err := o.Raw("select cve_id,cve_num,cve_url, cve_version, pack_name," + + "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity,"+ + "access_complexity,privilege_required,user_interaction,scope,confidentiality,integrity,availability,authentication," + + "cve_status,create_time from cve_origin_excel GROUP BY pack_name").QueryRows(&coe) + if err == nil && num > 0 { + logs.Info("cve_origin_excel 查询结果： ", num) + } else { + logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) + } + return coe, num, err +} + +func QueryOriginCveCenterData() ([]models.VulnCenter, int64, error) { + o := orm.NewOrm() + var coe []models.VulnCenter + num, err := o.Raw("select * from cve_vuln_center GROUP BY pack_name").QueryRows(&coe) + if err == nil && num > 0 { + logs.Info("cve_vuln_center 查询结果： ", num) + } else { + logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) + } + return coe, num, err +} + +func UpdateExcelDataPkName(originPackName, packName string) (id int64) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_origin_excel SET pack_name = ?" + + " WHERE pack_name = ?", packName, originPackName).QueryRow() + return +} + +func CreateRepo() { + //coe, num, err := QueryOriginCveExcelData() + coe, num, err := QueryOriginCveCenterData() + if err == nil && num > 0 { + for _, qc := range coe { + //PostRepo(qc.PackName) + PostGroupRepo(qc.PackName) + //CreateBrand(qc.PackName) + } + } +} + +func PostGroupRepo(packName string) error { + url := "https://gitee.com/api/v5/orgs/cve-test/repos" + accessToken := os.Getenv("GITEE_TOKEN") + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "name": "%s", + "has_issues": "true", + "has_wiki": "true", + "can_comment": "true", + "auto_init": "true", + "path": "%s", + "private": "false" + }`, accessToken, packName, packName) + logs.Info("isssue_body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("创建",packName,"失败, err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建issue 失败, err: ", ok, "url: ", url) + return errors.New("创建仓库失败") + } + errb := CreateBrand(packName) + logs.Info(errb) + return nil +} + +func PostRepo(packName string) error{ + url := "https://gitee.com/api/v5/user/repos" + accessToken := os.Getenv("GITEE_TOKEN") + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "name": "%s", + "has_issues": "true", + "has_wiki": "true", + "can_comment": "true", + "auto_init": "true", + "path": "%s", + "private": "false" + }`, accessToken, packName, packName) + logs.Info("isssue_body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("创建",packName,"失败, err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建issue 失败, err: ", ok, "url: ", url) + return errors.New("创建仓库失败") + } + errb := CreateBrand(packName) + logs.Info(errb) + return nil +} + +func CreateBrand(packName string) error{ + url := "https://gitee.com/api/v5/repos/cve-test/" + packName + "/branches" + accessToken := os.Getenv("GITEE_TOKEN") + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "refs": "master", + "branch_name": "openEuler-20.03-LTS" + }`, accessToken) + logs.Info("isssue_body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("创建分支：openEuler-20.03-LTS",packName,"失败, err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建分支：openEuler-20.03-LTS 失败, err: ", ok, "url: ", url) + return errors.New("创建分支失败") + } + + return nil +} + +<<<<<<< HEAD + +======= +func Init() { + CreateRepo() +} +>>>>>>> dcc533bfb3e46ca7de687eb69f20e183f2353c67 diff --git a/cve-vulner-manager/tests/default_test.go b/cve-vulner-manager/tests/default_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ebff2df1af1de6215676b8e2d3f73ac919cc4381 --- /dev/null +++ b/cve-vulner-manager/tests/default_test.go @@ -0,0 +1,38 @@ +package test + +import ( + "net/http" + "net/http/httptest" + "testing" + "runtime" + "path/filepath" + _ "cvevulner/routers" + + "github.com/astaxie/beego" + . "github.com/smartystreets/goconvey/convey" +) + +func init() { + _, file, _, _ := runtime.Caller(0) + apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".." + string(filepath.Separator)))) + beego.TestBeegoInit(apppath) +} + +// TestGet is a sample to run an endpoint test +func TestGet(t *testing.T) { + r, _ := http.NewRequest("GET", "/v1/object", nil) + w := httptest.NewRecorder() + beego.BeeApp.Handlers.ServeHTTP(w, r) + + beego.Trace("testing", "TestGet", "Code[%d]\n%s", w.Code, w.Body.String()) + + Convey("Subject: Test Station Endpoint\n", t, func() { + Convey("Status Code Should Be 200", func() { + So(w.Code, ShouldEqual, 200) + }) + Convey("The Result Should Not Be Empty", func() { + So(w.Body.Len(), ShouldBeGreaterThan, 0) + }) + }) +} + diff --git a/cve-vulner-manager/tests/excel_test.go b/cve-vulner-manager/tests/excel_test.go new file mode 100644 index 0000000000000000000000000000000000000000..87b3acf08d6fb5b6b3f505b7dbfbe25307819fea --- /dev/null +++ b/cve-vulner-manager/tests/excel_test.go @@ -0,0 +1,15 @@ +package test + +import ( + "cvevulner/taskhandler" + "testing" +) + +func TestGenerateCveExcel(t *testing.T) { + err := taskhandler.GenerateCveExcel("test.xlsx", "op-2020-10-", 1002, false) + if err != nil { + t.Error(err) + } +} + + diff --git a/cve-vulner-manager/tests/giteeissue_test.go b/cve-vulner-manager/tests/giteeissue_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e8f12064b05155b229fd37232dfaa4aa26645176 --- /dev/null +++ b/cve-vulner-manager/tests/giteeissue_test.go @@ -0,0 +1,20 @@ +package test + +import ( + "cvevulner/models" + "fmt" + "testing" +) + +func TestGetGiteOriginIssue(t *testing.T) { + //issue, err := models.GetGiteOriginIssue("CVE-2020-15719") + issue, err := models.GetGiteOriginIssue("CVE-2018-19541") + if err != nil { + t.Error(err) + } + loophole, err := issue.ParseToLoophole() + if err != nil { + t.Error(err) + } + t.Log(fmt.Sprintf("%#v", loophole)) +} diff --git a/cve-vulner-manager/tests/grabissue_test.go b/cve-vulner-manager/tests/grabissue_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d87e749115454523a2ea14753f74fa8c1e6cd48f --- /dev/null +++ b/cve-vulner-manager/tests/grabissue_test.go @@ -0,0 +1,16 @@ +package test + +import ( + "cvevulner/taskhandler" + "testing" +) + +func TestGrabIssueByOrg(t *testing.T) { + err := taskhandler.GrabIssueByOrg("bee5383db9aa3f19663b8aab82b76289", "cve-test") + if err != nil { + t.Error(err) + }else { + t.Log(err) + } +} + diff --git a/cve-vulner-manager/tests/issue_test.go b/cve-vulner-manager/tests/issue_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a171f08442fdcf935145fee9cbb71ed898f7fb7e --- /dev/null +++ b/cve-vulner-manager/tests/issue_test.go @@ -0,0 +1,48 @@ +package test + +import ( + "fmt" + "io/ioutil" + "net/http" + "sync" + "testing" +) + +var wt sync.WaitGroup + +func TestDeleteRepo(t *testing.T) { +/* repo, err := models.GetIssueTplGroupByRepo() + if err != nil { + t.Error(err) + } else {*/ + repo := []string{"javamail","libssh","libraw","flex","glibc","gstreamer","binutils","cairo","bouncycastle"} + for _, v := range repo { + go func(rep string) { + wt.Add(1) + defer wt.Done() + _ = deleteRepo("zhangjianjun_code", rep, "bee5383db9aa3f19663b8aab82b76289") + + }(v) + } + //} + wt.Wait() +} + +func deleteRepo(owner, repo, token string) error { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s?access_token=%s", owner, repo, token) + request, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + + rsp, err := http.DefaultClient.Do(request) + if err != nil { + return err + } + defer rsp.Body.Close() + + body, _ := ioutil.ReadAll(rsp.Body) + fmt.Println(string(body)) + + return nil +} diff --git a/cve-vulner-manager/tests/parsepayload_test.go b/cve-vulner-manager/tests/parsepayload_test.go new file mode 100644 index 0000000000000000000000000000000000000000..984cc44bc7e150e5237dd1a3e374c8406d5d29cc --- /dev/null +++ b/cve-vulner-manager/tests/parsepayload_test.go @@ -0,0 +1,22 @@ +package test + +import ( + "cvevulner/util" + "fmt" + "testing" +) + +func TestExtractCommentAnalysisAllValue(t *testing.T) { + tStr := `笑笑笑笑笑笑笑笑笑的方式是否士大夫十分 + 受影响版本排查(受影响/不受影响): + 1.openEuler-20-LTS:受影响 + 2.openEuler-22-LTS:受影响 + 3.openEuler-23-LTS:不受影响 + 4.openEuler-23-LTS: + 影响性分析说明:影响firefox的张昌使用 + openEuler评分: + BaseScore: 6.5 MEDIUM + Vector: CVSS:3.0/AV:N/AC:M/Au:N/UI:R/S:U/C:N/I:N/A:H` + value := util.ExtractCommentAnalysisAllValue(tStr) + t.Log(fmt.Sprintf("%#v", value)) +} diff --git a/cve-vulner-manager/util/file.go b/cve-vulner-manager/util/file.go new file mode 100644 index 0000000000000000000000000000000000000000..b4274dfe003357c6f771e54bb70fa08a53523f22 --- /dev/null +++ b/cve-vulner-manager/util/file.go @@ -0,0 +1,30 @@ +package util + +import "os" + +//IsExistPath Determine whether the file path exists +func IsExistPath(path string) (exist bool,err error) { + _,err = os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +//MakeDir make dir +func MakeDir(path string) error { + exist,err := IsExistPath(path) + if err != nil{ + return err + } + if !exist { + err := os.Mkdir(path, os.ModePerm) + if err != nil { + return err + } + } + return nil +} diff --git a/cve-vulner-manager/util/http.go b/cve-vulner-manager/util/http.go new file mode 100644 index 0000000000000000000000000000000000000000..7171a3b1e6931e7b2d23f523a2704d7c1bb69bc9 --- /dev/null +++ b/cve-vulner-manager/util/http.go @@ -0,0 +1,354 @@ +package util + +import ( + "bytes" + "cvevulner/common" + "encoding/json" + "fmt" + "github.com/astaxie/beego/logs" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" +) + +//RequestInfo http request information +type RequestInfo struct { + URL string + Data map[string]string //The data to be transmitted by the post, the key value must be both string + DataInterface map[string]interface{} +} + +//HTTPPatch patch request +func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { + req, err := http.NewRequest("PATCH", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() + if err != nil { + return nil, err + } + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("PATCH request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + defer resp.Body.Close() + logs.Info("response Status:", resp.Status, "url: ", url) + logs.Info("response Headers:", resp.Header, "url: ", url) + status, _ := strconv.Atoi(resp.Status) + if status > 300 { + logs.Error("Patch request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + fmt.Println("response Body:", string(body)) + if err != nil || body == nil { + logs.Error("PATCH failed, err: ", err, "body: ", requestBody) + return nil, err + } + logs.Info("PATCH successed!, body: ", string(body)) + var iss map[string]interface{} + err = json.Unmarshal(body, &iss) + if err != nil { + logs.Error(err, string(body)) + return nil, err + } + logs.Info(iss) + return iss, nil +} + +//HTTPPost post request +func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() + if err != nil { + return nil, err + } + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + defer resp.Body.Close() + logs.Info("response Status:", resp.Status, "url: ", url) + logs.Info("response Headers:", resp.Header, "url: ", url) + status, _ := strconv.Atoi(resp.Status) + if status > 300 { + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + fmt.Println("response Body:", string(body)) + if err != nil || body == nil { + logs.Error("post failed, err: ", err, "body: ", requestBody) + return nil, err + } + logs.Info("post successed!, body: ", string(body)) + var iss map[string]interface{} + err = json.Unmarshal(body, &iss) + if err != nil { + logs.Error(err, string(body)) + return nil, err + } + logs.Info(iss) + return iss, nil +} + +//HTTPPost1 post request +func HTTPPost1(url string, requestBody string) ([]map[string]interface{}, error) { + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() + if err != nil { + return nil, err + } + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + defer resp.Body.Close() + logs.Info("response Status:", resp.Status, "url: ", url) + logs.Info("response Headers:", resp.Header, "url: ", url) + status, _ := strconv.Atoi(resp.Status) + if status > 300 { + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + fmt.Println("response Body:", string(body)) + if err != nil || body == nil { + logs.Error("post failed, err: ", err, "body: ", requestBody) + return nil, err + } + logs.Info("post successed!, body: ", string(body)) + var iss []map[string]interface{} + err = json.Unmarshal(body, &iss) + if err != nil { + logs.Error(err, string(body)) + return nil, err + } + logs.Info(iss) + return iss, nil +} + +//PostURLEncoded request params encode +func PostURLEncoded(this RequestInfo) ([]byte, error) { + client := &http.Client{} + //post data to be submitted + DataURLVal := url.Values{} + for key, val := range this.Data { + DataURLVal.Add(key, val) + } + req, err := http.NewRequest("POST", this.URL, strings.NewReader(DataURLVal.Encode())) + if err != nil { + logs.Error(err) + return nil, err + } + //Camouflage head + req.Header.Set("Accept", "application/json") + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + + //Submit request + resp, err := client.Do(req) + if err != nil { + logs.Error(err) + return nil, err + } + defer resp.Body.Close() + //Read return value + result, err := ioutil.ReadAll(resp.Body) + if err != nil || result == nil { + logs.Error(err) + return nil, err + } + logs.Info(string(result)) + return result, nil +} + +//HTTPGet get request +func HTTPGet(url string) ([]map[string]interface{}, error) { + resp, err := http.Get(url) + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + if err != nil { + logs.Error("get error, url:", url, "error: ", err) + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + logs.Error("resp.StatusCode:", resp.StatusCode, ",url: ", url) + return nil, nil + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil || body == nil { + logs.Error("url:", url, ",err: ", err) + return nil, err + } + logs.Info("body: \n", string(body), "url: ", url) + var col []map[string]interface{} + err = json.Unmarshal(body, &col) + if err != nil { + logs.Error("HTTPGet,err: ", err, "url: ", url) + } + return col, nil +} + +//HTTPGetCom common get request +func HTTPGetCom(urls string) ([]byte, error) { + resp, err := http.Get(urls) + url := strings.Split(urls, "access_token") + if len(url) > 0 { + urls = url[0] + } + if err != nil { + logs.Error("get error, url:", urls, "error: ", err) + return nil, err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil || body == nil { + logs.Error(err) + return nil, err + } + logs.Info("body: \n", string(body), "url: ", urls) + return body, nil +} + +func HTTPDelCom(url, requestBody string) ([]map[string]interface{}, error) { + req, err := http.NewRequest("DELETE", url, bytes.NewBuffer([]byte(requestBody))) + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + defer common.Catchs() + if err != nil { + logs.Error("NewRequest, url: ", url, ",req: ", requestBody, ",err: ", err) + return nil, err + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("DELETE request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + defer resp.Body.Close() + logs.Info("response Status:", resp.Status, "url: ", url) + logs.Info("response Headers:", resp.Header, "url: ", url) + status, _ := strconv.Atoi(resp.Status) + if status > 300 { + logs.Error("DELETE request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + fmt.Println("response Body:", string(body)) + if err != nil || body == nil { + logs.Error("DELETE failed, err: ", err, "body: ", requestBody) + return nil, err + } + logs.Info("DELETE successed!, body: ", string(body)) + var iss []map[string]interface{} + err = json.Unmarshal(body, &iss) + if err != nil { + logs.Error(err, string(body)) + return nil, err + } + logs.Info(iss) + return iss, nil +} + +//HTTPPut patch request +func HTTPPut(url string, requestBody string) ([]map[string]interface{}, error) { + req, err := http.NewRequest("PUT", url, bytes.NewBuffer([]byte(requestBody))) + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + defer common.Catchs() + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("PATCH request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + defer resp.Body.Close() + logs.Info("response Status:", resp.Status, "url: ", url) + logs.Info("response Headers:", resp.Header, "url: ", url) + status, _ := strconv.Atoi(resp.Status) + if status > 300 { + logs.Error("Patch request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + fmt.Println("response Body:", string(body)) + if err != nil || body == nil || len(body) == 0 { + logs.Error("PUT failed, err: ", err, "body: ", requestBody) + return nil, err + } + logs.Info("PUT successed!, body: ", string(body)) + var iss []map[string]interface{} + err = json.Unmarshal(body, &iss) + if err != nil { + logs.Error(err, string(body)) + return nil, err + } + return iss, nil +} + +func HTTPPostCom(req map[string]interface{}, url string) ([]byte, error) { + logs.Info("post req_body: ", req) + bytesData, err := json.Marshal(req) + if err != nil { + logs.Error(err.Error()) + return bytesData, err + } + reader := bytes.NewReader(bytesData) + request, err := http.NewRequest("POST", url, reader) + urls := strings.Split(url, "access_token") + if len(urls) > 0 { + url = urls[0] + } + defer request.Body.Close() + if err != nil { + logs.Error(err.Error()) + return nil, err + } + request.Header.Set("Content-Type", "application/json;charset=UTF-8") + client := http.Client{} + resp, err := client.Do(request) + if err != nil { + logs.Error("url", url, ",req: ", req, ",err: ", err.Error()) + return nil, err + } + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("url", url, ",req: ", req, ",err: ", err.Error()) + return respBytes, err + } + logs.Info("respBytes", string(respBytes)) + return respBytes, err +} diff --git a/cve-vulner-manager/util/parsepayload.go b/cve-vulner-manager/util/parsepayload.go new file mode 100644 index 0000000000000000000000000000000000000000..05c615fff6bf87adf7b4e9cd94a8498d790b63ea --- /dev/null +++ b/cve-vulner-manager/util/parsepayload.go @@ -0,0 +1,603 @@ +package util + +import ( + "cvevulner/common" + "fmt" + "github.com/astaxie/beego/logs" + "regexp" + "sort" + "strings" + "sync" +) + +const ( + KwOpenGaussScore = "openGauss评分:" + KwMindSporeScore = "MindSpore评分:" + KwLooKengScore = "openLooKeng评分:" +) + +const ( + //KwAnalysisDesc Keywords for impact analysis + KwAnalysisDesc = "影响性分析说明:" + //KwOpenEulerScore Keywords scored by openEuler + KwOpenEulerScore = "openEuler评分:" + //KwEffectVersion Keywords of the affected version + KwEffectVersion = "受影响版本排查(受影响/不受影响):" + //KwAbiVersion Keywords of the abi version + KwAbiVersion = "修复是否涉及abi变化(是/否):" +) + +var ( + //LD Vulnerability description label + LD = "LD" + //IAD Impact analysis description label + IAD = "IAD" + //PA Principle analysis tags + PA = "PA" + //OES openEuler score tags + OES = "OES" + //OEV openEuler vector tags + OEV = "OEV" + //IV Affected version label + IV = "IV" + //CPMM Circumvention measures + CPMM = "CPMM" + //IW Impacted package + IW = "IW" + //CommentKeys Keyword parsed by the new version of comments + CommentKeys = []string{KwAnalysisDesc, KwOpenEulerScore, KwEffectVersion} + EulerCommentKeys = []string{KwAnalysisDesc, KwOpenEulerScore, KwEffectVersion, KwAbiVersion} +) + +var ( + //CvsScoreV3 nveScoreType v3 + CvsScoreV3 = "v3" + //CvsScoreV2 nveScoreType V2 + CvsScoreV2 = "v2" + // Regular match CVE information + RegexpCveNumVaule = regexp.MustCompile(`CVE-\d+-\d+`) + //RegexpCveNumber cveNum extract regexp + RegexpCveNumber = regexp.MustCompile(`漏洞编号[:：](?s:(.*?))漏洞归属组件[:：]`) + regexpCveNumberLink = regexp.MustCompile(`\[(.*?)\]\((.*?)\)`) + //RegexpCveComponents components extract regexp + RegexpCveComponents = regexp.MustCompile(`漏洞归属组件[:：](?s:(.*?))漏洞归属的?版本[:：]`) + //RegexpCveVersion cveVersion extract regexp + RegexpCveVersion = regexp.MustCompile(`漏洞归属的?版本[:：](?s:(.*?))CVSS [Vv][23].[0-9xX]分值[:：]`) + //RegexpCveScore cveScore extract regexp + RegexpCveScore = regexp.MustCompile(`CVSS [Vv][23].[0-9xX]分值[:：](?s:(.*?))漏洞[简描]述[:：]`) + //RegexpCveBriefDesc brief description extract regexp + RegexpCveBriefDesc = regexp.MustCompile(`漏洞[简描]述[:：](?s:(.*?))影响性分析说明[:：]`) + //RegexpCveBriefDesc new tpl brief description extract regexp + RegexpCveBriefDescNew = regexp.MustCompile(`漏洞[简描]述[:：](?s:(.*?))漏洞公开时间[:：]`) + //RegexpCveInfluencesDesc influences description regexp + RegexpCveInfluencesDesc = regexp.MustCompile(`影响性分析说明[:：](?s:(.*?))原理分析[:：]`) + //RegexpCveInfluencesDescNew new tpl influences description regexp + RegexpCveInfluencesDescNew = regexp.MustCompile(`影响性分析说明[:：](?s:(.*?))openEuler评分[:：]`) + //RegexpCvePrincipleDesc principle description regexp + RegexpCvePrincipleDesc = regexp.MustCompile(`原理分析[:：](?s:(.*?))openEuler评分[:：]`) + //RegexpCveOpScore openEuler score regexp + RegexpCveOpScore = regexp.MustCompile(`openEuler评分[:：](?s:(.*?))受影响版本[:：]`) + //RegexpCveOpScoreNew new tpl openEuler score regexp + RegexpCveOpScoreNew = regexp.MustCompile(`openEuler评分[:：](?s:(.*?))受影响版本排查\(受影响/不受影响\)[:：]`) + //RegexpCveInfluencesVersion influences version regexp + RegexpCveInfluencesVersion = regexp.MustCompile(`受影响版本[:：](?s:(.*?))规避方案或消减措施[:：]`) + //RegexpCveInfluencesVersionNew new tpl influences version regexp + RegexpCveInfluencesVersionNew = regexp.MustCompile(`受影响版本排查\(受影响/不受影响\)[:：](?s:(.*?))$`) + RegexpCveInfluencesVersionFixNew = regexp.MustCompile(`受影响版本排查\(受影响/不受影响\)[:：](?s:(.*?))三、漏洞修复`) + //RegexpCvePlannedMeasures the cve planned measures regexp + RegexpCvePlannedMeasures = regexp.MustCompile(`规避方案或消减措施[:：](?s:(.*?))受影响的包[:：]`) + //RegexpCvePlannedMeasures1 the second cve planned measures regexp + RegexpCvePlannedMeasures1 = regexp.MustCompile(`规避方案或消减措施[:：](?s:(.*?))$`) + //RegexpCveInfluencesPkg the cve influences package regexp + RegexpCveInfluencesPkg = regexp.MustCompile(`受影响的包[:：](?s:(.*?))$`) + //RegexpDigital digital regexp + RegexpDigital = regexp.MustCompile(`(\d){1,}(\.\d+)?`) + RegexpSpecialDigital = regexp.MustCompile(`(cvssv3.[0-9]|cvssv2.[0-9]|CVSSV3.[0-9]|CVSSV2.[0-9]|CVSS[:：]3.[0-9]|CVSS[:：]2.[0-9]|cvss[:：]3.[0-9]|cvss[:：]2.[0-9]|3.[0-9]/|2.[0-9]/|3.[0-9] /|2.[0-9] /)*`) //^((CVSS：3.0|CVSS：2.0|3.0/|2.0/|3.0 /|2.0 /).)*$ + //RegexpSpecialDigital = regexp.MustCompile(`(cvssv[1-9].[0-9]|CVSSV[1-9].[0-9]|CVSS[:：][1-9].[0-9]|cvss[:：][1-9].[0-9]|[1-9].[0-9]/|[1-9].[0-9] /)*`) //^((CVSS：3.0|CVSS：2.0|3.0/|2.0/|3.0 /|2.0 /).)*$ + RegexpVector = regexp.MustCompile(`AV:[NLAP](?s:(.*?))/A:[LNH]`) + RegexpVectorV2 = regexp.MustCompile(`AV:[LAN](?s:(.*))/Au:[MSN](?s:(.*))/A:[NPC]`) + RegexpScoreTypeV2 = regexp.MustCompile(`(?mi)^CVSS[vV]2.[0-9xX]\s*`) // CVSS V3.0分值: + RegexpScoreTypeV3 = regexp.MustCompile(`(?mi)^CVSS[vV]3.[0-9xX]\s*`) + RegexpIsNewTpl = regexp.MustCompile(`(?mi)^原理分析[：:]\s*`) + RegexpIsNewTpl2 = regexp.MustCompile(`(?mi)^规避方案或消减措施[:：]\s*`) + regexpEffectVersion = regexp.MustCompile(`(?mi)[\d]{1,}\.(.*?)[:：]受影响`) + regexpNoEffectVersion = regexp.MustCompile(`(?mi)[\d]{1,}\.(.*?)[:：]不受影响`) + regexpOtherEffectVersion = regexp.MustCompile(`(?mi)[\d]{1,}\.(.*?)[:：]$`) + regexpEffectAbiVersion = regexp.MustCompile(`(?mi)[\d]{1,}\.(.*?)[:：]是`) + regexpNoEffectAbiVersion = regexp.MustCompile(`(?mi)[\d]{1,}\.(.*?)[:：]否`) + RegexpIsAbiTpl = regexp.MustCompile(`(?mi)^(修复)?是否涉及abi变化\(是/否\)[：:]\s*`) + RegexpIsFixTpl = regexp.MustCompile(`(?mi)^三、漏洞修复\s*`) + //RegexpCveAbiVersionNew new tpl influences version regexp + RegexpCveAbiVersionNew = regexp.MustCompile(`受影响版本排查\(受影响/不受影响\)[:：](?s:(.*?))(修复)?是否涉及abi变化\(是/否\)[:：]`) + //RegexpCveAbiNew new tpl influences version regexp + RegexpCveAbiNew = regexp.MustCompile(`[修复]?是否涉及abi变化\(是/否\)[:：](?s:(.*?))$`) + RegexpCveAbiFixNew = regexp.MustCompile(`[修复]?是否涉及abi变化\(是/否\)[:：](?s:(.*?))三、漏洞修复`) + RegMatchCve = regexp.MustCompile("^CVE-[0-9]+-[0-9]+$") +) + +var ( + //VectorMapV2 the vector v2.0 map + VectorMapV2 map[string]map[string]string + //VectorMap the vector v3.0 map + VectorMap map[string]map[string]string + mutex sync.Mutex +) + +//CommentAnalysis issue comment analysis keyword and value container +type CommentAnalysis struct { + KeyName string + KeyIdx int + KeyValue string +} + +//CaSlice define the CommentAnalysis slice +type CaSlice []CommentAnalysis + +func (a CaSlice) Len() int { + return len(a) +} +func (a CaSlice) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} +func (a CaSlice) Less(i, j int) bool { + return a[j].KeyIdx > a[i].KeyIdx +} + +func init() { + VectorMap = make(map[string]map[string]string) + mAv := make(map[string]string) + mAv["N"] = "Network" + mAv["L"] = "Local" + mAv["A"] = "Adjacent" + mAv["P"] = "Physical" + VectorMap["AV"] = mAv + mAc := make(map[string]string) + mAc["H"] = "High" + mAc["L"] = "Low" + VectorMap["AC"] = mAc + mPr := make(map[string]string) + mPr["H"] = "High" + mPr["N"] = "None" + mPr["L"] = "Low" + VectorMap["PR"] = mPr + mUI := make(map[string]string) + mUI["N"] = "None" + mUI["R"] = "Required" + VectorMap["UI"] = mUI + mS := make(map[string]string) + mS["U"] = "Unchanged" + mS["C"] = "Changed" + VectorMap["S"] = mS + mC := make(map[string]string) + mC["H"] = "High" + mC["L"] = "Low" + mC["N"] = "None" + VectorMap["C"] = mC + mI := make(map[string]string) + mI["H"] = "High" + mI["N"] = "None" + mI["L"] = "Low" + VectorMap["I"] = mC + mA := make(map[string]string) + mA["H"] = "High" + mA["L"] = "Low" + mA["N"] = "None" + VectorMap["A"] = mA + VectorMapV2 = make(map[string]map[string]string) + //AV:L/AC:L/Au:N/C:C/I:C/A:C + mAv2 := make(map[string]string) + mAv2["L"] = "Local" + mAv2["A"] = "AdjacentNetwork" + mAv2["N"] = "Network" + VectorMapV2["AV"] = mAv2 + mAc2 := make(map[string]string) + mAc2["H"] = "High" + mAc2["M"] = "Medium" + mAc2["L"] = "Low" + VectorMapV2["AC"] = mAc2 + mAu := make(map[string]string) + mAu["M"] = "Multiple" + mAu["S"] = "Single" + mAu["N"] = "None" + VectorMapV2["Au"] = mAu + mCi := make(map[string]string) + mCi["N"] = "None" + mCi["P"] = "Partial" + mCi["C"] = "Complete" + VectorMapV2["C"] = mCi + mII := make(map[string]string) + mII["N"] = "None" + mII["P"] = "Partial" + mII["C"] = "Complete" + VectorMapV2["I"] = mII + mAi := make(map[string]string) + mAi["N"] = "None" + mAi["P"] = "Partial" + mAi["C"] = "Complete" + VectorMapV2["A"] = mAi +} + +//GenerateCommentAnalysis Generate analytical entities based on comments. +func GenerateCommentAnalysis(content string, organizationID int8) (ca CaSlice) { + if content == "" { + return + } + commentKeys := []string{} + if organizationID == 1 { + commentKeys = EulerCommentKeys + } else { + commentKeys = CommentKeys + } + for _, v := range commentKeys { + if strings.Contains(content, v) { + cm := CommentAnalysis{KeyName: v} + index := strings.Index(content, v) + cm.KeyIdx = index + ca = append(ca, cm) + } + } + //sort ca item by key index + if len(ca) > 0 { + //sort by index + sort.Sort(ca) + //extract keyword value + for k := range ca { + if k == len(ca)-1 { + start := ca[k].KeyIdx + len(ca[k].KeyName) + ca[k].KeyValue = content[start:] + } else { + start := ca[k].KeyIdx + len(ca[k].KeyName) + end := ca[k+1].KeyIdx + ca[k].KeyValue = content[start:end] + } + } + } + return ca +} + +//ParseCommentContent extract comment content based on tags. +func ParseCommentContent(content string, label string) (res string, ok bool) { + ret := regexp.MustCompile(genCommentRegexpStr(label)) + sm := ret.FindAllStringSubmatch(content, 1) + if len(sm) > 0 { + res = sm[0][1] + ok = true + } + return +} + +//ParseCommentVector extract vector from issue comment +func ParseCommentVector(content string) string { + sm := RegexpVector.Find([]byte(content)) + return string(sm) +} + +//ExtractVector extract vector from issue body +func ExtractVector(body, scoreType string) string { + if body == "" { + return body + } + if scoreType == CvsScoreV2 { + rvs := RegexpVectorV2.Find([]byte(body)) + if rv := string(rvs); rv != "" { + return rv + } + } else { + rvs := RegexpVector.Find([]byte(body)) + if rv := string(rvs); rv != "" { + return rv + } + } + + return "" +} + +//ReadVMValue get vector v3 value from the vector map by keyword +func ReadVMValue(kStr string) (value string) { + if kStr == "" { + return "" + } + if !strings.Contains(kStr, ":") { + return "" + } + kStr = TrimString(kStr) + sKs := strings.Split(kStr, ":") + if len(sKs) != 2 { + return "" + } + mutex.Lock() + defer mutex.Unlock() + if _, ok := VectorMap[sKs[0]]; ok { + value = VectorMap[sKs[0]][sKs[1]] + } + return +} + +//ReadVMValueV2 get vector v2 value from the vector map by keyword +func ReadVMValueV2(kStr string) (value string) { + if kStr == "" { + return "" + } + if !strings.Contains(kStr, ":") { + return "" + } + kStr = TrimString(kStr) + sKs := strings.Split(kStr, ":") + if len(sKs) != 2 { + return "" + } + mutex.Lock() + defer mutex.Unlock() + if _, ok := VectorMapV2[sKs[0]]; ok { + value = VectorMapV2[sKs[0]][sKs[1]] + } + return +} + +//VctToMap Convert vector string value to map +func VctToMap(vct string) (vctMap map[string]string, ok bool) { + if vct == "" { + return nil, false + } + sp := strings.Split(vct, "/") + if len(sp) < 1 { + return nil, false + } + vMap := make(map[string]string) + for _, v := range sp { + spv := strings.Split(v, ":") + if len(spv) != 2 { + continue + } + vMap[spv[0]] = v + } + if len(vMap) > 0 { + return vMap, true + } + return nil, false + +} + +//ParseCommentWithAllLabel extract comment value with custom label +func ParseCommentWithAllLabel(content string) map[string]string { + res := make(map[string]string, 0) + s, ok := ParseCommentContent(content, IAD) + if ok { + res["cve_analysis"] = s + } + s, ok = ParseCommentContent(content, PA) + if ok { + res["principle_analysis"] = s + } + s, ok = ParseCommentContent(content, OES) + if ok { + res["openeuler_score"] = TrimString(s) + } + s, ok = ParseCommentContent(content, OEV) + if ok { + vector := ParseCommentVector(s) + if vector != "" { + res["openeuler_vector"] = s + } + } + s, ok = ParseCommentContent(content, IV) + if ok { + res["affected_version"] = s + } + s, ok = ParseCommentContent(content, CPMM) + if ok { + res["solution"] = s + } + s, ok = ParseCommentContent(content, IW) + if ok { + res["issue_package"] = s + } + + return res +} + +//ExtractCommentAnalysisAllValue Extract all value by issue comment +func ExtractCommentAnalysisAllValue(content string, organizationID int8) map[string]string { + res := make(map[string]string, 0) + ca := GenerateCommentAnalysis(content, organizationID) + if len(ca) > 0 { + value, ext := ExtractCommentValue(ca, KwAnalysisDesc) + if ext { + res["cve_analysis"] = TrimStringNR(value) + } + value, ext = ExtractCommentValue(ca, KwEffectVersion) + if ext { + value = ExtractCommentEffectVersion(value) + res["affected_version"] = value + } + value, ext = ExtractCommentValue(ca, KwOpenEulerScore) + if ext { + score, vector := ExtractCommentOpenEulerScore(value) + if score != "" { + res["openeuler_score"] = score + } + if vector != "" { + res["openeuler_vector"] = vector + } + } + if organizationID == 1 { + value, ext = ExtractCommentValue(ca, KwAbiVersion) + if ext { + value = ExtractCommentAbiVersion(value) + res["abi_version"] = value + } + } + } + return res +} + +//ExtractCommentEffectVersion Extract the affected version from the issue comment +func ExtractCommentEffectVersion(str string) string { + str = strings.Trim(str, " ") + str = strings.ReplaceAll(str, " ", "") + var res []string + match := regexpEffectVersion.FindAllStringSubmatch(str, -1) + match2 := regexpNoEffectVersion.FindAllStringSubmatch(str, -1) + if len(match) > 0 || len(match2) > 0 { + if len(match) > 0 { + for _, v := range match { + if len(v) > 1 { + tmpV := TrimString(v[1]) + ":受影响" + tmpV = common.BranchVersionRep(tmpV) + res = append(res, tmpV) + } + } + } + if len(match2) > 0 { + for _, v := range match2 { + if len(v) > 1 { + tmpV := TrimString(v[1]) + ":不受影响" + tmpV = common.BranchVersionRep(tmpV) + res = append(res, tmpV) + } + } + } + } + match3 := regexpOtherEffectVersion.FindAllStringSubmatch(str, -1) + if len(match3) > 0 { + for _, v := range match3 { + if len(v) > 1 { + tmpV := TrimString(v[1]) + ":" + tmpV = common.BranchVersionRep(tmpV) + res = append(res, tmpV) + } + } + } + if len(res) > 0 { + return strings.Join(res, ",") + } + return "" +} + +//ExtractCommentAbiVersion Extract the abi version from the issue comment +func ExtractCommentAbiVersion(str string) string { + str = strings.Trim(str, " ") + str = strings.ReplaceAll(str, " ", "") + var res []string + match := regexpEffectAbiVersion.FindAllStringSubmatch(str, -1) + match2 := regexpNoEffectAbiVersion.FindAllStringSubmatch(str, -1) + if len(match) > 0 || len(match2) > 0 { + if len(match) > 0 { + for _, v := range match { + if len(v) > 1 { + tmpV := TrimString(v[1]) + ":是" + tmpV = common.BranchVersionRep(tmpV) + res = append(res, tmpV) + } + } + } + if len(match2) > 0 { + for _, v := range match2 { + if len(v) > 1 { + tmpV := TrimString(v[1]) + ":否" + tmpV = common.BranchVersionRep(tmpV) + res = append(res, tmpV) + } + } + } + } + match3 := regexpOtherEffectVersion.FindAllStringSubmatch(str, -1) + if len(match3) > 0 { + for _, v := range match3 { + if len(v) > 1 { + tmpV := TrimString(v[1]) + ":" + tmpV = common.BranchVersionRep(tmpV) + res = append(res, tmpV) + } + } + } + if len(res) > 0 { + return strings.Join(res, ",") + } + return "" +} + +//ExtractCommentValue Get the value of CaSlice by keyword +func ExtractCommentValue(ca CaSlice, keyWord string) (string, bool) { + for _, v := range ca { + if v.KeyName == keyWord { + return v.KeyValue, true + } + } + return "", false +} + +//ExtractCommentOpenEulerScore Extract openEuler score from issue comment +func ExtractCommentOpenEulerScore(str string) (score, vector string) { + str = TrimString(str) + score = ExtractDigital(str) + vector = ExtractVector(str, CvsScoreV3) + if vector == "" { + vector = ExtractVector(str, CvsScoreV2) + } + return +} + +func genCommentRegexpStr(label string) string { + return fmt.Sprintf(`\[%s\](?s:(.*?))\[/%s\]`, label, label) +} + +//TrimString Remove the \n \r \t spaces in the string +func TrimString(str string) string { + str = strings.Replace(str, " ", "", -1) + str = strings.Replace(str, "\n", "", -1) + str = strings.Replace(str, "\r", "", -1) + str = strings.Replace(str, "\t", "", -1) + return str +} + +//TrimStringNR Remove the \n \r in the string +func TrimStringNR(str string) string { + str = strings.Replace(str, "\n", "", -1) + str = strings.Replace(str, "\r", "", -1) + str = strings.Replace(str, "\t", "", -1) + return str +} + +//ExtractDigital remove "cvss 3.0" or "cvss 2.0" +func RemoveSpecialDigital(body string) string { + if body == "" { + return body + } + sds := RegexpSpecialDigital.ReplaceAllString(body, "") + logs.Info("openEuler_value: ", sds) + if len(sds) > 0 { + return sds + } + return "" +} + +//ExtractDigital Get number in string +func ExtractDigital(body string) string { + if body == "" { + return body + } + scoreStr := "" + tempStr := RemoveSpecialDigital(body) + if tempStr != "" && len(tempStr) > 0 { + scoreStr = tempStr + } else { + scoreStr = body + } + sds := RegexpDigital.FindAllStringSubmatch(scoreStr, -1) + if len(sds) > 0 { + return sds[0][0] + } + return "" +} + +//GetCveNumber Extract cveNum from the issue body cveNumber link +func GetCveNumber(ov string) string { + if v := regexpCveNumberLink.Find([]byte(ov)); len(v) > 0 { + sv := string(v) + start := strings.Index(sv, "[") + 1 + end := strings.Index(sv, "]") + return sv[start:end] + } + return ov +} diff --git a/cve-vulner-manager/util/time.go b/cve-vulner-manager/util/time.go new file mode 100644 index 0000000000000000000000000000000000000000..bf895fcdc9eeafa919828df2bfea3fafef00dc97 --- /dev/null +++ b/cve-vulner-manager/util/time.go @@ -0,0 +1,22 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "time" +) + +//TimeStrToInt parse time string to unix nano +func TimeStrToInt(ts, layout string) int64 { + if ts == "" { + return 0 + } + if layout == "" { + layout = "2006-01-02 15:04:05" + } + t, err := time.ParseInLocation(layout, ts, time.Local) + if err != nil { + logs.Error(err) + return 0 + } + return t.Unix() +}