From 86881759551b95da38bb763106cf6deb95b0be26 Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:12:19 +0100 Subject: conf: add git2 dependence --- Cargo.lock | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++ 2 files changed, 103 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bcdd80e..e035a1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,6 +469,9 @@ name = "cc" version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -913,6 +916,7 @@ dependencies = [ "config", "deadpool-postgres", "dotenv", + "git2", "hex", "md-5", "regex", @@ -920,6 +924,7 @@ dependencies = [ "slog", "slog-async", "slog-term", + "time", "tokio-pg-mapper", "tokio-pg-mapper-derive", "tokio-postgres", @@ -932,6 +937,21 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +[[package]] +name = "git2" +version = "0.13.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d250f5f82326884bd39c2853577e70a121775db76818ffa452ed1e80de12986" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "h2" version = "0.2.7" @@ -1077,6 +1097,15 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +[[package]] +name = "jobserver" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +dependencies = [ + "libc", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1118,6 +1147,46 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" +[[package]] +name = "libgit2-sys" +version = "0.12.18+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da6a42da88fc37ee1ecda212ffa254c25713532980005d5f7c0b0fbe7e6e885" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0186af0d8f171ae6b9c4c90ec51898bad5d08a2d5e470903a50d9ad8959cbee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -1338,6 +1407,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.11.1" @@ -1445,6 +1533,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + [[package]] name = "postgres-protocol" version = "0.5.3" @@ -2129,6 +2223,12 @@ dependencies = [ "serde 1.0.124", ] +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + [[package]] name = "version_check" version = "0.9.2" diff --git a/Cargo.toml b/Cargo.toml index 42b2318..cafe258 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,10 @@ dotenv = "0.15.0" config = "0.10.1" serde = { version = "1.0.104", features = ["derive"] } chrono = { version = "0.4", features = ["serde"] } +time = "0.1" uuid = { version = "0.8.2", features = ["serde", "v4"] } regex = "1" md-5 = "0.9.1" hex = "0.4.3" + +git2 = "0.13" -- cgit v1.2.3-18-g5258 From 8fe649cb5f12f990ae067d6c80293c0dd241731c Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:12:46 +0100 Subject: chore: add git error type and equals macro --- src/errors.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/errors.rs b/src/errors.rs index 8140cfe..1efb082 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -4,11 +4,12 @@ use serde::Serialize; use std::fmt; use tokio_postgres::error::Error; -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq)] pub enum AppErrorType { DbError, NotFoundError, AuthorizationError, + GitError, } #[derive(Debug)] @@ -72,6 +73,7 @@ impl ResponseError for AppError { AppErrorType::DbError => StatusCode::INTERNAL_SERVER_ERROR, AppErrorType::NotFoundError => StatusCode::NOT_FOUND, AppErrorType::AuthorizationError => StatusCode::UNAUTHORIZED, + AppErrorType::GitError => StatusCode::BAD_REQUEST, } } -- cgit v1.2.3-18-g5258 From 23090f94334a662e11a3120fc84cf7a1e7e983b0 Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:13:31 +0100 Subject: feat(commit): create new commits from vec --- src/commit/models.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/commit/models.rs b/src/commit/models.rs index 8b81a15..2f1536b 100644 --- a/src/commit/models.rs +++ b/src/commit/models.rs @@ -93,4 +93,50 @@ impl Commit { }), } } + + /// Create commits from an array + pub async fn create( + pool: Pool, + commits: Vec, + ) -> Result, AppError> { + let client = get_client(pool.clone()).await.unwrap(); + let mut raw_query = "INSERT INTO commit VALUES".to_string(); + + for commit in commits { + let tree = match commit.tree { + Some(t) => format!("'{}'", t), + None => "NULL".to_string(), + }; + raw_query += &format!( + "('{}', {}, '{}', '{}', '{}', '{}', '{}', '{}', '{}'),", + commit.hash, + tree, + commit.text, + commit.date, + commit.author_email, + commit.author_name, + commit.committer_email, + commit.committer_name, + commit.repository_url + )[..] + } + + // Remove the last `,` + let _ = raw_query.pop(); + + // TODO: write query with &commits and parameter. Need to implement + // ToSql trait for `Commit` model + // let statement = client.prepare(&query[..]).await?; + // client.query(&statement, &[&commits] + + let statement = client.prepare(&raw_query[..]).await?; + let result = client + .query(&statement, &[]) + .await? + .iter() + .map(|row| Commit::from_row_ref(row).unwrap()) + .collect::>(); + + Ok(result) + } } -- cgit v1.2.3-18-g5258 From 4289e4961b6ba48a58e3719206d80b9cc9e795be Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:13:58 +0100 Subject: feat: add 'git' module --- src/git.rs | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 src/git.rs diff --git a/src/git.rs b/src/git.rs new file mode 100644 index 0000000..76f9a4e --- /dev/null +++ b/src/git.rs @@ -0,0 +1,142 @@ +use crate::commit::models::Commit; +use chrono::{DateTime, Local}; +use git2::{BranchType, Error, Repository, Time}; + +use std::fs::remove_dir_all; + +/// Return the temporary folder of the git repository +fn get_tmp_dir(repo_name: &String) -> String { + format!("/tmp/{}", &repo_name) +} + +/// Clone a repository `repo` in a temporary folder. It uses only GitHub at +/// this moment. +pub fn clone_repo(repo_name: &String) -> Result { + let url: &str = + &format!("https://github.com/{}", &repo_name).to_owned()[..]; + let tmp_dir: String = get_tmp_dir(&repo_name); + + let cloned = Repository::clone(url, tmp_dir)?; + + Ok(cloned) +} + +/// Check if a `branch` exists inside the repository and then set the head of +/// `repo` to that branch +pub fn get_branch(repo: &Repository, branch: &str) -> Result<(), Error> { + if let Err(e) = repo.find_branch(&branch, BranchType::Local) { + return Err(e); + } + + repo.set_head(&format!("refs/heads/{}", branch).to_owned()[..]) +} + +/// Get a `git2::Commit` and returns a date in RFC2822 format +pub fn get_date(commit: &git2::Commit) -> String { + let t: Time = commit.time(); + let (offset, sign) = match t.offset_minutes() { + n if n < 0 => (-n, '-'), + n => (n, '+'), + }; + let (hours, minutes) = (offset / 60, offset % 60); + let ts = + time::Timespec::new(t.seconds() + (t.offset_minutes() as i64) * 60, 0); + let time = time::at(ts); + + let result = format!( + "{} {}{:02}{:02}", + time.strftime("%a, %e %b %Y %T").unwrap(), + sign, + hours, + minutes + ); + + result +} + +/// Get a `git2::Commit` and returns a valid `Commit` to upload to the database +fn get_commit(gcommit: &git2::Commit, repo_name: &String) -> Commit { + let hash = gcommit.id().to_string(); + let tree = match gcommit.parent_id(0) { + Ok(parent) => Some(parent.to_string()), + Err(_) => None, + }; + + let mut text = "".to_string(); + for line in String::from_utf8_lossy(gcommit.message_bytes()).lines() { + text += line; + text += "\n"; + } + + // Remove the last "\n" + let _ = text.pop(); + + let date: DateTime = + match DateTime::parse_from_rfc2822(&get_date(&gcommit)) { + Ok(date) => date.into(), + Err(e) => { + // This is a real problem! + // TODO: manage this error + panic!(e) + } + }; + let author_email = gcommit.author().email().unwrap().to_string(); + let author_name = gcommit.author().name().unwrap().to_string(); + let committer_email = gcommit.committer().email().unwrap().to_string(); + let committer_name = gcommit.committer().name().unwrap().to_string(); + + Commit { + hash, + tree, + text, + date, + author_email, + author_name, + committer_email, + committer_name, + repository_url: repo_name.clone(), + } +} + +/// Get all commits from a Git repository. Returns an array of `Commit`. +/// First, clone the repo into a temporary folder. +/// Then, open the repository. +/// Then, get commits +/// Finally, remove the temporary folder +pub fn repo_commits(repo_name: &String) -> Result, Error> { + // Try to clone the repo. If it returns an error, it's useless to go ahead: + // raises an error. + let repo = match clone_repo(&repo_name) { + Ok(r) => r, + Err(e) => { + return Err(e); + } + }; + + if let Err(e) = get_branch(&repo, "main") { + return Err(e); + } + + let mut revwalk = repo.revwalk()?; + let head = repo.head().unwrap().target().unwrap(); + + if let Err(e) = revwalk.push(head) { + return Err(e); + } + + let mut commits: Vec = vec![]; + for commit in revwalk { + let hash = repo.find_commit(commit?)?; + commits.push(get_commit(&hash, &repo_name)); + } + + if let Err(_) = remove_dir_all(get_tmp_dir(&repo_name)) { + return Err(git2::Error::new( + git2::ErrorCode::GenericError, + git2::ErrorClass::Os, + "Temporary clone not deleted", + )); + } + + Ok(commits) +} -- cgit v1.2.3-18-g5258 From 5f4f01791d26003889afbb7179633ca98d5647e1 Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:14:15 +0100 Subject: feat: 'git' module imported --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index 6f20201..2e499f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,8 @@ mod db; mod errors; mod helpers; +mod git; + mod branch; mod commit; mod email; -- cgit v1.2.3-18-g5258 From 214137b65413daff684cfe8df4d544ea0cb78516 Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:14:32 +0100 Subject: feat: add Commit(s) when repository is created --- src/repository/models.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/repository/models.rs b/src/repository/models.rs index e3d8456..5b7b445 100644 --- a/src/repository/models.rs +++ b/src/repository/models.rs @@ -1,5 +1,8 @@ +use crate::commit::models::Commit; use crate::db::get_client; +use crate::email::models::{Email, EmailData}; use crate::errors::{AppError, AppErrorType}; +use crate::git; use crate::helpers::name_of_git_repository; use chrono::NaiveDateTime; @@ -9,6 +12,7 @@ use tokio_pg_mapper::FromTokioPostgresRow; use tokio_pg_mapper_derive::PostgresMapper; use uuid::Uuid; +use std::collections::HashSet; use std::net::SocketAddr; #[derive(Serialize, Deserialize, PostgresMapper)] @@ -185,7 +189,47 @@ impl Repository { .map(|row| Repository::from_row_ref(&row).unwrap()); match repo { - Some(repo) => Ok(repo), + Some(repo) => { + let commits = match git::repo_commits(&repo_name) { + Ok(c) => c, + Err(e) => { + return Err(AppError { + message: Some( + format!( + "Repository couldn't be created now: {:?}", + e + ) + .to_string(), + ), + cause: Some("Repository clone".to_string()), + error_type: AppErrorType::GitError, + }) + } + }; + + let mut emails: HashSet = HashSet::new(); + for commit in &commits { + emails.insert(commit.author_email.clone()); + emails.insert(commit.committer_email.clone()); + } + for email in emails { + if let Err(e) = + Email::create(pool.clone(), &EmailData { email }).await + { + if e.error_type == AppErrorType::DbError { + return Err(e); + } + } + } + + let commits_result = + Commit::create(pool.clone(), commits).await; + if let Err(e) = commits_result { + panic!("{}", e); + } + + Ok(repo) + } None => Err(AppError { message: Some("Error creating a new repository".to_string()), cause: Some("Unknown error".to_string()), -- cgit v1.2.3-18-g5258 From 7bdd6d8f2d18022c34986a6eea4620d8eb6dcb3a Mon Sep 17 00:00:00 2001 From: Santo Cariotti Date: Thu, 18 Mar 2021 12:18:19 +0100 Subject: fix: handle error for commit creation on repos --- src/repository/models.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repository/models.rs b/src/repository/models.rs index 5b7b445..de82d45 100644 --- a/src/repository/models.rs +++ b/src/repository/models.rs @@ -225,7 +225,7 @@ impl Repository { let commits_result = Commit::create(pool.clone(), commits).await; if let Err(e) = commits_result { - panic!("{}", e); + return Err(e); } Ok(repo) -- cgit v1.2.3-18-g5258