summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSanto Cariotti <santo@dcariotti.me>2021-03-18 12:19:25 +0100
committerGitHub <noreply@github.com>2021-03-18 12:19:25 +0100
commit77715d93b2112b7d231db541ecce6ddca5c762c7 (patch)
treec3718080abcbe0a6cc52735bcc6726952c6494a8
parentb8e22e372cb947771aa122767a4405afa208226e (diff)
parent7bdd6d8f2d18022c34986a6eea4620d8eb6dcb3a (diff)
Merge pull request #18 from gico-net/feat/git2
Create commits from a new Git Repository
-rw-r--r--Cargo.lock100
-rw-r--r--Cargo.toml3
-rw-r--r--src/commit/models.rs46
-rw-r--r--src/errors.rs4
-rw-r--r--src/git.rs142
-rw-r--r--src/main.rs2
-rw-r--r--src/repository/models.rs46
7 files changed, 341 insertions, 2 deletions
diff --git a/Cargo.lock b/Cargo.lock
index bcdd80e..e035a1b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -469,6 +469,9 @@ name = "cc"
version = "1.0.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+dependencies = [
+ "jobserver",
+]
[[package]]
name = "cfg-if"
@@ -913,6 +916,7 @@ dependencies = [
"config",
"deadpool-postgres",
"dotenv",
+ "git2",
"hex",
"md-5",
"regex",
@@ -920,6 +924,7 @@ dependencies = [
"slog",
"slog-async",
"slog-term",
+ "time",
"tokio-pg-mapper",
"tokio-pg-mapper-derive",
"tokio-postgres",
@@ -933,6 +938,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
[[package]]
+name = "git2"
+version = "0.13.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d250f5f82326884bd39c2853577e70a121775db76818ffa452ed1e80de12986"
+dependencies = [
+ "bitflags",
+ "libc",
+ "libgit2-sys",
+ "log",
+ "openssl-probe",
+ "openssl-sys",
+ "url",
+]
+
+[[package]]
name = "h2"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1078,6 +1098,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
[[package]]
+name = "jobserver"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2"
+dependencies = [
+ "libc",
+]
+
+[[package]]
name = "kernel32-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1119,6 +1148,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a"
[[package]]
+name = "libgit2-sys"
+version = "0.12.18+1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3da6a42da88fc37ee1ecda212ffa254c25713532980005d5f7c0b0fbe7e6e885"
+dependencies = [
+ "cc",
+ "libc",
+ "libssh2-sys",
+ "libz-sys",
+ "openssl-sys",
+ "pkg-config",
+]
+
+[[package]]
+name = "libssh2-sys"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0186af0d8f171ae6b9c4c90ec51898bad5d08a2d5e470903a50d9ad8959cbee"
+dependencies = [
+ "cc",
+ "libc",
+ "libz-sys",
+ "openssl-sys",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
name = "linked-hash-map"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1339,6 +1408,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
+name = "openssl-probe"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
name = "parking_lot"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1446,6 +1534,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
+name = "pkg-config"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
+
+[[package]]
name = "postgres-protocol"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2130,6 +2224,12 @@ dependencies = [
]
[[package]]
+name = "vcpkg"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
+
+[[package]]
name = "version_check"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 42b2318..cafe258 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -23,7 +23,10 @@ dotenv = "0.15.0"
config = "0.10.1"
serde = { version = "1.0.104", features = ["derive"] }
chrono = { version = "0.4", features = ["serde"] }
+time = "0.1"
uuid = { version = "0.8.2", features = ["serde", "v4"] }
regex = "1"
md-5 = "0.9.1"
hex = "0.4.3"
+
+git2 = "0.13"
diff --git a/src/commit/models.rs b/src/commit/models.rs
index 8b81a15..2f1536b 100644
--- a/src/commit/models.rs
+++ b/src/commit/models.rs
@@ -93,4 +93,50 @@ impl Commit {
}),
}
}
+
+ /// Create commits from an array
+ pub async fn create(
+ pool: Pool,
+ commits: Vec<Commit>,
+ ) -> Result<Vec<Commit>, AppError> {
+ let client = get_client(pool.clone()).await.unwrap();
+ let mut raw_query = "INSERT INTO commit VALUES".to_string();
+
+ for commit in commits {
+ let tree = match commit.tree {
+ Some(t) => format!("'{}'", t),
+ None => "NULL".to_string(),
+ };
+ raw_query += &format!(
+ "('{}', {}, '{}', '{}', '{}', '{}', '{}', '{}', '{}'),",
+ commit.hash,
+ tree,
+ commit.text,
+ commit.date,
+ commit.author_email,
+ commit.author_name,
+ commit.committer_email,
+ commit.committer_name,
+ commit.repository_url
+ )[..]
+ }
+
+ // Remove the last `,`
+ let _ = raw_query.pop();
+
+ // TODO: write query with &commits and parameter. Need to implement
+ // ToSql trait for `Commit` model
+ // let statement = client.prepare(&query[..]).await?;
+ // client.query(&statement, &[&commits]
+
+ let statement = client.prepare(&raw_query[..]).await?;
+ let result = client
+ .query(&statement, &[])
+ .await?
+ .iter()
+ .map(|row| Commit::from_row_ref(row).unwrap())
+ .collect::<Vec<Commit>>();
+
+ Ok(result)
+ }
}
diff --git a/src/errors.rs b/src/errors.rs
index 8140cfe..1efb082 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -4,11 +4,12 @@ use serde::Serialize;
use std::fmt;
use tokio_postgres::error::Error;
-#[derive(Debug)]
+#[derive(Debug, Eq, PartialEq)]
pub enum AppErrorType {
DbError,
NotFoundError,
AuthorizationError,
+ GitError,
}
#[derive(Debug)]
@@ -72,6 +73,7 @@ impl ResponseError for AppError {
AppErrorType::DbError => StatusCode::INTERNAL_SERVER_ERROR,
AppErrorType::NotFoundError => StatusCode::NOT_FOUND,
AppErrorType::AuthorizationError => StatusCode::UNAUTHORIZED,
+ AppErrorType::GitError => StatusCode::BAD_REQUEST,
}
}
diff --git a/src/git.rs b/src/git.rs
new file mode 100644
index 0000000..76f9a4e
--- /dev/null
+++ b/src/git.rs
@@ -0,0 +1,142 @@
+use crate::commit::models::Commit;
+use chrono::{DateTime, Local};
+use git2::{BranchType, Error, Repository, Time};
+
+use std::fs::remove_dir_all;
+
+/// Return the temporary folder of the git repository
+fn get_tmp_dir(repo_name: &String) -> String {
+ format!("/tmp/{}", &repo_name)
+}
+
+/// Clone a repository `repo` in a temporary folder. It uses only GitHub at
+/// this moment.
+pub fn clone_repo(repo_name: &String) -> Result<Repository, Error> {
+ let url: &str =
+ &format!("https://github.com/{}", &repo_name).to_owned()[..];
+ let tmp_dir: String = get_tmp_dir(&repo_name);
+
+ let cloned = Repository::clone(url, tmp_dir)?;
+
+ Ok(cloned)
+}
+
+/// Check if a `branch` exists inside the repository and then set the head of
+/// `repo` to that branch
+pub fn get_branch(repo: &Repository, branch: &str) -> Result<(), Error> {
+ if let Err(e) = repo.find_branch(&branch, BranchType::Local) {
+ return Err(e);
+ }
+
+ repo.set_head(&format!("refs/heads/{}", branch).to_owned()[..])
+}
+
+/// Get a `git2::Commit` and returns a date in RFC2822 format
+pub fn get_date(commit: &git2::Commit) -> String {
+ let t: Time = commit.time();
+ let (offset, sign) = match t.offset_minutes() {
+ n if n < 0 => (-n, '-'),
+ n => (n, '+'),
+ };
+ let (hours, minutes) = (offset / 60, offset % 60);
+ let ts =
+ time::Timespec::new(t.seconds() + (t.offset_minutes() as i64) * 60, 0);
+ let time = time::at(ts);
+
+ let result = format!(
+ "{} {}{:02}{:02}",
+ time.strftime("%a, %e %b %Y %T").unwrap(),
+ sign,
+ hours,
+ minutes
+ );
+
+ result
+}
+
+/// Get a `git2::Commit` and returns a valid `Commit` to upload to the database
+fn get_commit(gcommit: &git2::Commit, repo_name: &String) -> Commit {
+ let hash = gcommit.id().to_string();
+ let tree = match gcommit.parent_id(0) {
+ Ok(parent) => Some(parent.to_string()),
+ Err(_) => None,
+ };
+
+ let mut text = "".to_string();
+ for line in String::from_utf8_lossy(gcommit.message_bytes()).lines() {
+ text += line;
+ text += "\n";
+ }
+
+ // Remove the last "\n"
+ let _ = text.pop();
+
+ let date: DateTime<Local> =
+ match DateTime::parse_from_rfc2822(&get_date(&gcommit)) {
+ Ok(date) => date.into(),
+ Err(e) => {
+ // This is a real problem!
+ // TODO: manage this error
+ panic!(e)
+ }
+ };
+ let author_email = gcommit.author().email().unwrap().to_string();
+ let author_name = gcommit.author().name().unwrap().to_string();
+ let committer_email = gcommit.committer().email().unwrap().to_string();
+ let committer_name = gcommit.committer().name().unwrap().to_string();
+
+ Commit {
+ hash,
+ tree,
+ text,
+ date,
+ author_email,
+ author_name,
+ committer_email,
+ committer_name,
+ repository_url: repo_name.clone(),
+ }
+}
+
+/// Get all commits from a Git repository. Returns an array of `Commit`.
+/// First, clone the repo into a temporary folder.
+/// Then, open the repository.
+/// Then, get commits
+/// Finally, remove the temporary folder
+pub fn repo_commits(repo_name: &String) -> Result<Vec<Commit>, Error> {
+ // Try to clone the repo. If it returns an error, it's useless to go ahead:
+ // raises an error.
+ let repo = match clone_repo(&repo_name) {
+ Ok(r) => r,
+ Err(e) => {
+ return Err(e);
+ }
+ };
+
+ if let Err(e) = get_branch(&repo, "main") {
+ return Err(e);
+ }
+
+ let mut revwalk = repo.revwalk()?;
+ let head = repo.head().unwrap().target().unwrap();
+
+ if let Err(e) = revwalk.push(head) {
+ return Err(e);
+ }
+
+ let mut commits: Vec<Commit> = vec![];
+ for commit in revwalk {
+ let hash = repo.find_commit(commit?)?;
+ commits.push(get_commit(&hash, &repo_name));
+ }
+
+ if let Err(_) = remove_dir_all(get_tmp_dir(&repo_name)) {
+ return Err(git2::Error::new(
+ git2::ErrorCode::GenericError,
+ git2::ErrorClass::Os,
+ "Temporary clone not deleted",
+ ));
+ }
+
+ Ok(commits)
+}
diff --git a/src/main.rs b/src/main.rs
index 6f20201..2e499f9 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,6 +3,8 @@ mod db;
mod errors;
mod helpers;
+mod git;
+
mod branch;
mod commit;
mod email;
diff --git a/src/repository/models.rs b/src/repository/models.rs
index e3d8456..de82d45 100644
--- a/src/repository/models.rs
+++ b/src/repository/models.rs
@@ -1,5 +1,8 @@
+use crate::commit::models::Commit;
use crate::db::get_client;
+use crate::email::models::{Email, EmailData};
use crate::errors::{AppError, AppErrorType};
+use crate::git;
use crate::helpers::name_of_git_repository;
use chrono::NaiveDateTime;
@@ -9,6 +12,7 @@ use tokio_pg_mapper::FromTokioPostgresRow;
use tokio_pg_mapper_derive::PostgresMapper;
use uuid::Uuid;
+use std::collections::HashSet;
use std::net::SocketAddr;
#[derive(Serialize, Deserialize, PostgresMapper)]
@@ -185,7 +189,47 @@ impl Repository {
.map(|row| Repository::from_row_ref(&row).unwrap());
match repo {
- Some(repo) => Ok(repo),
+ Some(repo) => {
+ let commits = match git::repo_commits(&repo_name) {
+ Ok(c) => c,
+ Err(e) => {
+ return Err(AppError {
+ message: Some(
+ format!(
+ "Repository couldn't be created now: {:?}",
+ e
+ )
+ .to_string(),
+ ),
+ cause: Some("Repository clone".to_string()),
+ error_type: AppErrorType::GitError,
+ })
+ }
+ };
+
+ let mut emails: HashSet<String> = HashSet::new();
+ for commit in &commits {
+ emails.insert(commit.author_email.clone());
+ emails.insert(commit.committer_email.clone());
+ }
+ for email in emails {
+ if let Err(e) =
+ Email::create(pool.clone(), &EmailData { email }).await
+ {
+ if e.error_type == AppErrorType::DbError {
+ return Err(e);
+ }
+ }
+ }
+
+ let commits_result =
+ Commit::create(pool.clone(), commits).await;
+ if let Err(e) = commits_result {
+ return Err(e);
+ }
+
+ Ok(repo)
+ }
None => Err(AppError {
message: Some("Error creating a new repository".to_string()),
cause: Some("Unknown error".to_string()),