migrations.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678
  1. // Copyright 2015 The Gogs Authors. All rights reserved.
  2. // Use of this source code is governed by a MIT-style
  3. // license that can be found in the LICENSE file.
  4. package migrations
  5. import (
  6. "bytes"
  7. "encoding/json"
  8. "fmt"
  9. "io/ioutil"
  10. "os"
  11. "path"
  12. "path/filepath"
  13. "strings"
  14. "time"
  15. "github.com/Unknwon/com"
  16. "github.com/go-xorm/xorm"
  17. gouuid "github.com/satori/go.uuid"
  18. "gopkg.in/ini.v1"
  19. "github.com/gogits/gogs/modules/base"
  20. "github.com/gogits/gogs/modules/log"
  21. "github.com/gogits/gogs/modules/setting"
  22. )
  23. const _MIN_DB_VER = 4
  24. type Migration interface {
  25. Description() string
  26. Migrate(*xorm.Engine) error
  27. }
  28. type migration struct {
  29. description string
  30. migrate func(*xorm.Engine) error
  31. }
  32. func NewMigration(desc string, fn func(*xorm.Engine) error) Migration {
  33. return &migration{desc, fn}
  34. }
  35. func (m *migration) Description() string {
  36. return m.description
  37. }
  38. func (m *migration) Migrate(x *xorm.Engine) error {
  39. return m.migrate(x)
  40. }
  41. // The version table. Should have only one row with id==1
  42. type Version struct {
  43. ID int64 `xorm:"pk autoincr"`
  44. Version int64
  45. }
  46. // This is a sequence of migrations. Add new migrations to the bottom of the list.
  47. // If you want to "retire" a migration, remove it from the top of the list and
  48. // update _MIN_VER_DB accordingly
  49. var migrations = []Migration{
  50. NewMigration("fix locale file load panic", fixLocaleFileLoadPanic), // V4 -> V5:v0.6.0
  51. NewMigration("trim action compare URL prefix", trimCommitActionAppUrlPrefix), // V5 -> V6:v0.6.3
  52. NewMigration("generate issue-label from issue", issueToIssueLabel), // V6 -> V7:v0.6.4
  53. NewMigration("refactor attachment table", attachmentRefactor), // V7 -> V8:v0.6.4
  54. NewMigration("rename pull request fields", renamePullRequestFields), // V8 -> V9:v0.6.16
  55. NewMigration("clean up migrate repo info", cleanUpMigrateRepoInfo), // V9 -> V10:v0.6.20
  56. NewMigration("generate rands and salt for organizations", generateOrgRandsAndSalt), // V10 -> V11:v0.8.5
  57. NewMigration("convert date to unix timestamp", convertDateToUnix), // V11 -> V12:v0.9.2
  58. NewMigration("convert LDAP UseSSL option to SecurityProtocol", ldapUseSSLToSecurityProtocol), // V12 -> V13:v0.9.37
  59. }
  60. // Migrate database to current version
  61. func Migrate(x *xorm.Engine) error {
  62. if err := x.Sync(new(Version)); err != nil {
  63. return fmt.Errorf("sync: %v", err)
  64. }
  65. currentVersion := &Version{ID: 1}
  66. has, err := x.Get(currentVersion)
  67. if err != nil {
  68. return fmt.Errorf("get: %v", err)
  69. } else if !has {
  70. // If the version record does not exist we think
  71. // it is a fresh installation and we can skip all migrations.
  72. currentVersion.Version = int64(_MIN_DB_VER + len(migrations))
  73. if _, err = x.InsertOne(currentVersion); err != nil {
  74. return fmt.Errorf("insert: %v", err)
  75. }
  76. }
  77. v := currentVersion.Version
  78. if _MIN_DB_VER > v {
  79. log.Fatal(4, `Gogs no longer supports auto-migration from your previously installed version.
  80. Please try to upgrade to a lower version (>= v0.6.0) first, then upgrade to current version.`)
  81. return nil
  82. }
  83. if int(v-_MIN_DB_VER) > len(migrations) {
  84. // User downgraded Gogs.
  85. currentVersion.Version = int64(len(migrations) + _MIN_DB_VER)
  86. _, err = x.Id(1).Update(currentVersion)
  87. return err
  88. }
  89. for i, m := range migrations[v-_MIN_DB_VER:] {
  90. log.Info("Migration: %s", m.Description())
  91. if err = m.Migrate(x); err != nil {
  92. return fmt.Errorf("do migrate: %v", err)
  93. }
  94. currentVersion.Version = v + int64(i) + 1
  95. if _, err = x.Id(1).Update(currentVersion); err != nil {
  96. return err
  97. }
  98. }
  99. return nil
  100. }
  101. func sessionRelease(sess *xorm.Session) {
  102. if !sess.IsCommitedOrRollbacked {
  103. sess.Rollback()
  104. }
  105. sess.Close()
  106. }
  107. func fixLocaleFileLoadPanic(_ *xorm.Engine) error {
  108. cfg, err := ini.Load(setting.CustomConf)
  109. if err != nil {
  110. return fmt.Errorf("load custom config: %v", err)
  111. }
  112. cfg.DeleteSection("i18n")
  113. if err = cfg.SaveTo(setting.CustomConf); err != nil {
  114. return fmt.Errorf("save custom config: %v", err)
  115. }
  116. setting.Langs = strings.Split(strings.Replace(strings.Join(setting.Langs, ","), "fr-CA", "fr-FR", 1), ",")
  117. return nil
  118. }
  119. func trimCommitActionAppUrlPrefix(x *xorm.Engine) error {
  120. type PushCommit struct {
  121. Sha1 string
  122. Message string
  123. AuthorEmail string
  124. AuthorName string
  125. }
  126. type PushCommits struct {
  127. Len int
  128. Commits []*PushCommit
  129. CompareUrl string
  130. }
  131. type Action struct {
  132. ID int64 `xorm:"pk autoincr"`
  133. Content string `xorm:"TEXT"`
  134. }
  135. results, err := x.Query("SELECT `id`,`content` FROM `action` WHERE `op_type`=?", 5)
  136. if err != nil {
  137. return fmt.Errorf("select commit actions: %v", err)
  138. }
  139. sess := x.NewSession()
  140. defer sessionRelease(sess)
  141. if err = sess.Begin(); err != nil {
  142. return err
  143. }
  144. var pushCommits *PushCommits
  145. for _, action := range results {
  146. actID := com.StrTo(string(action["id"])).MustInt64()
  147. if actID == 0 {
  148. continue
  149. }
  150. pushCommits = new(PushCommits)
  151. if err = json.Unmarshal(action["content"], pushCommits); err != nil {
  152. return fmt.Errorf("unmarshal action content[%d]: %v", actID, err)
  153. }
  154. infos := strings.Split(pushCommits.CompareUrl, "/")
  155. if len(infos) <= 4 {
  156. continue
  157. }
  158. pushCommits.CompareUrl = strings.Join(infos[len(infos)-4:], "/")
  159. p, err := json.Marshal(pushCommits)
  160. if err != nil {
  161. return fmt.Errorf("marshal action content[%d]: %v", actID, err)
  162. }
  163. if _, err = sess.Id(actID).Update(&Action{
  164. Content: string(p),
  165. }); err != nil {
  166. return fmt.Errorf("update action[%d]: %v", actID, err)
  167. }
  168. }
  169. return sess.Commit()
  170. }
  171. func issueToIssueLabel(x *xorm.Engine) error {
  172. type IssueLabel struct {
  173. ID int64 `xorm:"pk autoincr"`
  174. IssueID int64 `xorm:"UNIQUE(s)"`
  175. LabelID int64 `xorm:"UNIQUE(s)"`
  176. }
  177. issueLabels := make([]*IssueLabel, 0, 50)
  178. results, err := x.Query("SELECT `id`,`label_ids` FROM `issue`")
  179. if err != nil {
  180. if strings.Contains(err.Error(), "no such column") ||
  181. strings.Contains(err.Error(), "Unknown column") {
  182. return nil
  183. }
  184. return fmt.Errorf("select issues: %v", err)
  185. }
  186. for _, issue := range results {
  187. issueID := com.StrTo(issue["id"]).MustInt64()
  188. // Just in case legacy code can have duplicated IDs for same label.
  189. mark := make(map[int64]bool)
  190. for _, idStr := range strings.Split(string(issue["label_ids"]), "|") {
  191. labelID := com.StrTo(strings.TrimPrefix(idStr, "$")).MustInt64()
  192. if labelID == 0 || mark[labelID] {
  193. continue
  194. }
  195. mark[labelID] = true
  196. issueLabels = append(issueLabels, &IssueLabel{
  197. IssueID: issueID,
  198. LabelID: labelID,
  199. })
  200. }
  201. }
  202. sess := x.NewSession()
  203. defer sessionRelease(sess)
  204. if err = sess.Begin(); err != nil {
  205. return err
  206. }
  207. if err = sess.Sync2(new(IssueLabel)); err != nil {
  208. return fmt.Errorf("sync2: %v", err)
  209. } else if _, err = sess.Insert(issueLabels); err != nil {
  210. return fmt.Errorf("insert issue-labels: %v", err)
  211. }
  212. return sess.Commit()
  213. }
  214. func attachmentRefactor(x *xorm.Engine) error {
  215. type Attachment struct {
  216. ID int64 `xorm:"pk autoincr"`
  217. UUID string `xorm:"uuid INDEX"`
  218. // For rename purpose.
  219. Path string `xorm:"-"`
  220. NewPath string `xorm:"-"`
  221. }
  222. results, err := x.Query("SELECT * FROM `attachment`")
  223. if err != nil {
  224. return fmt.Errorf("select attachments: %v", err)
  225. }
  226. attachments := make([]*Attachment, 0, len(results))
  227. for _, attach := range results {
  228. if !com.IsExist(string(attach["path"])) {
  229. // If the attachment is already missing, there is no point to update it.
  230. continue
  231. }
  232. attachments = append(attachments, &Attachment{
  233. ID: com.StrTo(attach["id"]).MustInt64(),
  234. UUID: gouuid.NewV4().String(),
  235. Path: string(attach["path"]),
  236. })
  237. }
  238. sess := x.NewSession()
  239. defer sessionRelease(sess)
  240. if err = sess.Begin(); err != nil {
  241. return err
  242. }
  243. if err = sess.Sync2(new(Attachment)); err != nil {
  244. return fmt.Errorf("Sync2: %v", err)
  245. }
  246. // Note: Roll back for rename can be a dead loop,
  247. // so produces a backup file.
  248. var buf bytes.Buffer
  249. buf.WriteString("# old path -> new path\n")
  250. // Update database first because this is where error happens the most often.
  251. for _, attach := range attachments {
  252. if _, err = sess.Id(attach.ID).Update(attach); err != nil {
  253. return err
  254. }
  255. attach.NewPath = path.Join(setting.AttachmentPath, attach.UUID[0:1], attach.UUID[1:2], attach.UUID)
  256. buf.WriteString(attach.Path)
  257. buf.WriteString("\t")
  258. buf.WriteString(attach.NewPath)
  259. buf.WriteString("\n")
  260. }
  261. // Then rename attachments.
  262. isSucceed := true
  263. defer func() {
  264. if isSucceed {
  265. return
  266. }
  267. dumpPath := path.Join(setting.LogRootPath, "attachment_path.dump")
  268. ioutil.WriteFile(dumpPath, buf.Bytes(), 0666)
  269. fmt.Println("Fail to rename some attachments, old and new paths are saved into:", dumpPath)
  270. }()
  271. for _, attach := range attachments {
  272. if err = os.MkdirAll(path.Dir(attach.NewPath), os.ModePerm); err != nil {
  273. isSucceed = false
  274. return err
  275. }
  276. if err = os.Rename(attach.Path, attach.NewPath); err != nil {
  277. isSucceed = false
  278. return err
  279. }
  280. }
  281. return sess.Commit()
  282. }
  283. func renamePullRequestFields(x *xorm.Engine) (err error) {
  284. type PullRequest struct {
  285. ID int64 `xorm:"pk autoincr"`
  286. PullID int64 `xorm:"INDEX"`
  287. PullIndex int64
  288. HeadBarcnh string
  289. IssueID int64 `xorm:"INDEX"`
  290. Index int64
  291. HeadBranch string
  292. }
  293. if err = x.Sync(new(PullRequest)); err != nil {
  294. return fmt.Errorf("sync: %v", err)
  295. }
  296. results, err := x.Query("SELECT `id`,`pull_id`,`pull_index`,`head_barcnh` FROM `pull_request`")
  297. if err != nil {
  298. if strings.Contains(err.Error(), "no such column") {
  299. return nil
  300. }
  301. return fmt.Errorf("select pull requests: %v", err)
  302. }
  303. sess := x.NewSession()
  304. defer sessionRelease(sess)
  305. if err = sess.Begin(); err != nil {
  306. return err
  307. }
  308. var pull *PullRequest
  309. for _, pr := range results {
  310. pull = &PullRequest{
  311. ID: com.StrTo(pr["id"]).MustInt64(),
  312. IssueID: com.StrTo(pr["pull_id"]).MustInt64(),
  313. Index: com.StrTo(pr["pull_index"]).MustInt64(),
  314. HeadBranch: string(pr["head_barcnh"]),
  315. }
  316. if pull.Index == 0 {
  317. continue
  318. }
  319. if _, err = sess.Id(pull.ID).Update(pull); err != nil {
  320. return err
  321. }
  322. }
  323. return sess.Commit()
  324. }
  325. func cleanUpMigrateRepoInfo(x *xorm.Engine) (err error) {
  326. type (
  327. User struct {
  328. ID int64 `xorm:"pk autoincr"`
  329. LowerName string
  330. }
  331. Repository struct {
  332. ID int64 `xorm:"pk autoincr"`
  333. OwnerID int64
  334. LowerName string
  335. }
  336. )
  337. repos := make([]*Repository, 0, 25)
  338. if err = x.Where("is_mirror=?", false).Find(&repos); err != nil {
  339. return fmt.Errorf("select all non-mirror repositories: %v", err)
  340. }
  341. var user *User
  342. for _, repo := range repos {
  343. user = &User{ID: repo.OwnerID}
  344. has, err := x.Get(user)
  345. if err != nil {
  346. return fmt.Errorf("get owner of repository[%d - %d]: %v", repo.ID, repo.OwnerID, err)
  347. } else if !has {
  348. continue
  349. }
  350. configPath := filepath.Join(setting.RepoRootPath, user.LowerName, repo.LowerName+".git/config")
  351. // In case repository file is somehow missing.
  352. if !com.IsFile(configPath) {
  353. continue
  354. }
  355. cfg, err := ini.Load(configPath)
  356. if err != nil {
  357. return fmt.Errorf("open config file: %v", err)
  358. }
  359. cfg.DeleteSection("remote \"origin\"")
  360. if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
  361. return fmt.Errorf("save config file: %v", err)
  362. }
  363. }
  364. return nil
  365. }
  366. func generateOrgRandsAndSalt(x *xorm.Engine) (err error) {
  367. type User struct {
  368. ID int64 `xorm:"pk autoincr"`
  369. Rands string `xorm:"VARCHAR(10)"`
  370. Salt string `xorm:"VARCHAR(10)"`
  371. }
  372. orgs := make([]*User, 0, 10)
  373. if err = x.Where("type=1").And("rands=''").Find(&orgs); err != nil {
  374. return fmt.Errorf("select all organizations: %v", err)
  375. }
  376. sess := x.NewSession()
  377. defer sessionRelease(sess)
  378. if err = sess.Begin(); err != nil {
  379. return err
  380. }
  381. for _, org := range orgs {
  382. org.Rands = base.GetRandomString(10)
  383. org.Salt = base.GetRandomString(10)
  384. if _, err = sess.Id(org.ID).Update(org); err != nil {
  385. return err
  386. }
  387. }
  388. return sess.Commit()
  389. }
  390. type TAction struct {
  391. ID int64 `xorm:"pk autoincr"`
  392. CreatedUnix int64
  393. }
  394. func (t *TAction) TableName() string { return "action" }
  395. type TNotice struct {
  396. ID int64 `xorm:"pk autoincr"`
  397. CreatedUnix int64
  398. }
  399. func (t *TNotice) TableName() string { return "notice" }
  400. type TComment struct {
  401. ID int64 `xorm:"pk autoincr"`
  402. CreatedUnix int64
  403. }
  404. func (t *TComment) TableName() string { return "comment" }
  405. type TIssue struct {
  406. ID int64 `xorm:"pk autoincr"`
  407. DeadlineUnix int64
  408. CreatedUnix int64
  409. UpdatedUnix int64
  410. }
  411. func (t *TIssue) TableName() string { return "issue" }
  412. type TMilestone struct {
  413. ID int64 `xorm:"pk autoincr"`
  414. DeadlineUnix int64
  415. ClosedDateUnix int64
  416. }
  417. func (t *TMilestone) TableName() string { return "milestone" }
  418. type TAttachment struct {
  419. ID int64 `xorm:"pk autoincr"`
  420. CreatedUnix int64
  421. }
  422. func (t *TAttachment) TableName() string { return "attachment" }
  423. type TLoginSource struct {
  424. ID int64 `xorm:"pk autoincr"`
  425. CreatedUnix int64
  426. UpdatedUnix int64
  427. }
  428. func (t *TLoginSource) TableName() string { return "login_source" }
  429. type TPull struct {
  430. ID int64 `xorm:"pk autoincr"`
  431. MergedUnix int64
  432. }
  433. func (t *TPull) TableName() string { return "pull_request" }
  434. type TRelease struct {
  435. ID int64 `xorm:"pk autoincr"`
  436. CreatedUnix int64
  437. }
  438. func (t *TRelease) TableName() string { return "release" }
  439. type TRepo struct {
  440. ID int64 `xorm:"pk autoincr"`
  441. CreatedUnix int64
  442. UpdatedUnix int64
  443. }
  444. func (t *TRepo) TableName() string { return "repository" }
  445. type TMirror struct {
  446. ID int64 `xorm:"pk autoincr"`
  447. UpdatedUnix int64
  448. NextUpdateUnix int64
  449. }
  450. func (t *TMirror) TableName() string { return "mirror" }
  451. type TPublicKey struct {
  452. ID int64 `xorm:"pk autoincr"`
  453. CreatedUnix int64
  454. UpdatedUnix int64
  455. }
  456. func (t *TPublicKey) TableName() string { return "public_key" }
  457. type TDeployKey struct {
  458. ID int64 `xorm:"pk autoincr"`
  459. CreatedUnix int64
  460. UpdatedUnix int64
  461. }
  462. func (t *TDeployKey) TableName() string { return "deploy_key" }
  463. type TAccessToken struct {
  464. ID int64 `xorm:"pk autoincr"`
  465. CreatedUnix int64
  466. UpdatedUnix int64
  467. }
  468. func (t *TAccessToken) TableName() string { return "access_token" }
  469. type TUser struct {
  470. ID int64 `xorm:"pk autoincr"`
  471. CreatedUnix int64
  472. UpdatedUnix int64
  473. }
  474. func (t *TUser) TableName() string { return "user" }
  475. type TWebhook struct {
  476. ID int64 `xorm:"pk autoincr"`
  477. CreatedUnix int64
  478. UpdatedUnix int64
  479. }
  480. func (t *TWebhook) TableName() string { return "webhook" }
  481. func convertDateToUnix(x *xorm.Engine) (err error) {
  482. log.Info("This migration could take up to minutes, please be patient.")
  483. type Bean struct {
  484. ID int64 `xorm:"pk autoincr"`
  485. Created time.Time
  486. Updated time.Time
  487. Merged time.Time
  488. Deadline time.Time
  489. ClosedDate time.Time
  490. NextUpdate time.Time
  491. }
  492. var tables = []struct {
  493. name string
  494. cols []string
  495. bean interface{}
  496. }{
  497. {"action", []string{"created"}, new(TAction)},
  498. {"notice", []string{"created"}, new(TNotice)},
  499. {"comment", []string{"created"}, new(TComment)},
  500. {"issue", []string{"deadline", "created", "updated"}, new(TIssue)},
  501. {"milestone", []string{"deadline", "closed_date"}, new(TMilestone)},
  502. {"attachment", []string{"created"}, new(TAttachment)},
  503. {"login_source", []string{"created", "updated"}, new(TLoginSource)},
  504. {"pull_request", []string{"merged"}, new(TPull)},
  505. {"release", []string{"created"}, new(TRelease)},
  506. {"repository", []string{"created", "updated"}, new(TRepo)},
  507. {"mirror", []string{"updated", "next_update"}, new(TMirror)},
  508. {"public_key", []string{"created", "updated"}, new(TPublicKey)},
  509. {"deploy_key", []string{"created", "updated"}, new(TDeployKey)},
  510. {"access_token", []string{"created", "updated"}, new(TAccessToken)},
  511. {"user", []string{"created", "updated"}, new(TUser)},
  512. {"webhook", []string{"created", "updated"}, new(TWebhook)},
  513. }
  514. for _, table := range tables {
  515. log.Info("Converting table: %s", table.name)
  516. if err = x.Sync2(table.bean); err != nil {
  517. return fmt.Errorf("Sync [table: %s]: %v", table.name, err)
  518. }
  519. offset := 0
  520. for {
  521. beans := make([]*Bean, 0, 100)
  522. if err = x.Sql(fmt.Sprintf("SELECT * FROM `%s` ORDER BY id ASC LIMIT 100 OFFSET %d",
  523. table.name, offset)).Find(&beans); err != nil {
  524. return fmt.Errorf("select beans [table: %s, offset: %d]: %v", table.name, offset, err)
  525. }
  526. log.Trace("Table [%s]: offset: %d, beans: %d", table.name, offset, len(beans))
  527. if len(beans) == 0 {
  528. break
  529. }
  530. offset += 100
  531. baseSQL := "UPDATE `" + table.name + "` SET "
  532. for _, bean := range beans {
  533. valSQLs := make([]string, 0, len(table.cols))
  534. for _, col := range table.cols {
  535. fieldSQL := ""
  536. fieldSQL += col + "_unix = "
  537. switch col {
  538. case "deadline":
  539. if bean.Deadline.IsZero() {
  540. continue
  541. }
  542. fieldSQL += com.ToStr(bean.Deadline.Unix())
  543. case "created":
  544. fieldSQL += com.ToStr(bean.Created.Unix())
  545. case "updated":
  546. fieldSQL += com.ToStr(bean.Updated.Unix())
  547. case "closed_date":
  548. fieldSQL += com.ToStr(bean.ClosedDate.Unix())
  549. case "merged":
  550. fieldSQL += com.ToStr(bean.Merged.Unix())
  551. case "next_update":
  552. fieldSQL += com.ToStr(bean.NextUpdate.Unix())
  553. }
  554. valSQLs = append(valSQLs, fieldSQL)
  555. }
  556. if len(valSQLs) == 0 {
  557. continue
  558. }
  559. if _, err = x.Exec(baseSQL + strings.Join(valSQLs, ",") + " WHERE id = " + com.ToStr(bean.ID)); err != nil {
  560. return fmt.Errorf("update bean [table: %s, id: %d]: %v", table.name, bean.ID, err)
  561. }
  562. }
  563. }
  564. }
  565. return nil
  566. }