Commit 7d306d55 authored by Evgeni Sladkovskii's avatar Evgeni Sladkovskii

pushing client

parents
Pipeline #7223 canceled with stages
/target/
!.mvn/wrapper/maven-wrapper.jar
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/build/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>nl.trifork</groupId>
<artifactId>axondb-backup-client</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>axondb-backup-client</name>
<description>AxonDB backup client</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>26.0-jre</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.axonframework</groupId>
<artifactId>axon-spring-boot-starter</artifactId>
<version>3.3.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.axoniq</groupId>
<artifactId>axondb-client</artifactId>
<version>1.2.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.axoniq</groupId>
<artifactId>axondb-grpc-proto</artifactId>
<version>1.2.3</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>axondb-backup-client</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
####Small backup client for AxonDB
Idea behind this client is to schedule AxonDB backups and store them in predefined location.
Client periodically calls AxonDB to check if there is new closed event files.
If there is one or more it creates a controlDB backup file and copies it and new events files to preconfigured locations.
This client should be ran near AxonDB since controlDB backups are stored in root dir.
In case of ordinary machine/virtual machine deployment just run it as a separate process.
In case of docker use `nohup java -jar backup.jar --spring.config.location=/axondb-backup.yml &` in your run script to run it in background to AxonDB.
####Restore
To restore copy all files events backups to corresponding folders in AxonDB.
Also copy last controlDB dump and name it `${db-name}.mv.db` depending on your config.
After start AxonDB should load all restored events and load all new ones from other replicas.
####Manual Testing
This project has AxnDb load test that can be used to generate events in your AxonDB instance to force it's backup.
\ No newline at end of file
package nl.trifork.axondbbackupclient;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.AutoConfigurationPackage;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.web.client.RestTemplate;
@SpringBootApplication
@EnableScheduling
@AutoConfigurationPackage
public class AxonDbBackupClientApplication {
public static void main(String[] args) {
SpringApplication.run(AxonDbBackupClientApplication.class, args);
}
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
}
package nl.trifork.axondbbackupclient;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@Data
@ConfigurationProperties(prefix = "backup")
@Component
@Slf4j
public class BackupConfig {
private String axondbHost;
private String eventsUrl;
private String snapshotsUrl;
private String dbDumpUrl;
private String axondbToken;
private String eventsBackupPath;
private String snapshotsBackupPath;
private String dbDumpBackupPath;
@PostConstruct
public void print() {
log.info(toString());
}
}
package nl.trifork.axondbbackupclient.axondb_client;
import lombok.RequiredArgsConstructor;
import nl.trifork.axondbbackupclient.BackupConfig;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import static org.springframework.http.HttpMethod.GET;
import static org.springframework.http.HttpMethod.POST;
@Service
@RequiredArgsConstructor
public class AxonDbRestClient {
private final RestTemplate restTemplate;
private final BackupConfig config;
public String[] getClosedEventSegments() {
return getReadySegments(eventsUrl());
}
public String[] getClosedSnapshotsSegments() {
return getReadySegments(snapshotsUrl());
}
private String[] getReadySegments(String url) {
ResponseEntity<String[]> segments = restTemplate.exchange(url, GET, reqEntity(), String[].class);
return segments.getBody();
}
public String createControlDbDump() {
ResponseEntity<String> dbDumpPath = restTemplate.exchange(dbDumpUrl(), POST, reqEntity(), String.class);
return dbDumpPath.getBody();
}
private String eventsUrl() {
return config.getAxondbHost() + config.getEventsUrl();
}
private String snapshotsUrl() {
return config.getAxondbHost() + config.getSnapshotsUrl();
}
private String dbDumpUrl() {
return config.getAxondbHost() + config.getDbDumpUrl();
}
private HttpEntity reqEntity() {
HttpHeaders headers = new HttpHeaders();
headers.add("Access-Token", config.getAxondbToken());
HttpEntity<String> entity = new HttpEntity<>(headers);
return entity;
}
}
package nl.trifork.axondbbackupclient.backup_services;
import com.google.common.base.Stopwatch;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import nl.trifork.axondbbackupclient.BackupConfig;
import nl.trifork.axondbbackupclient.axondb_client.AxonDbRestClient;
import org.springframework.stereotype.Service;
import java.io.File;
import static com.google.common.base.Stopwatch.createStarted;
@Service
@Slf4j
@RequiredArgsConstructor
public class ControlDBBackupService {
private final BackupConfig config;
private final AxonDbRestClient axonDbClient;
private final FileIoService fileService;
public void backup() {
Stopwatch stopwatch = createStarted();
File dbDump = new File(axonDbClient.createControlDbDump());
if (dbDump.exists()) {
try {
fileService.copy(dbDump, config.getDbDumpBackupPath());
log.info("Created and copied db dump in {}", stopwatch);
} finally {
dbDump.delete();
}
}
}
}
package nl.trifork.axondbbackupclient.backup_services;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import static org.apache.commons.io.FileUtils.copyFile;
@Slf4j
@Service
public class FileIoService {
public void copy(File file, String destinationDir) {
try {
String destinationFilePath = destinationDir + file.getName();
File destinationFile = new File(destinationFilePath);
if (destinationFile.exists()) {
log.warn("Destination file already present {}", destinationFile);
return;
}
File tmpDestFile = new File(destinationFilePath + ".tmp");
copyFile(file, tmpDestFile);
tmpDestFile.renameTo(destinationFile);
} catch (IOException e) {
log.error("Failed to copy file {}", file, e);
}
}
}
package nl.trifork.axondbbackupclient.backup_services;
import com.google.common.base.Stopwatch;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import nl.trifork.axondbbackupclient.BackupConfig;
import nl.trifork.axondbbackupclient.axondb_client.AxonDbRestClient;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.File;
import static com.google.common.base.Stopwatch.createStarted;
import static java.util.Arrays.stream;
@Service
@RequiredArgsConstructor
@Slf4j
public class SegmentsBackupService {
private final BackupConfig config;
private final AxonDbRestClient axonDbClient;
private final ControlDBBackupService dbBackupService;
private final FileIoService fileService;
@Scheduled(fixedDelayString = "${backup.fixedDelayMs}", initialDelayString = "${backup.initialDelayMs}")
public void backup() {
if (hasNewSegments()) {
dbBackupService.backup();
backupEvents();
backupSnapshots();
} else {
log.info("No new segments");
}
}
private boolean hasNewSegments() {
String[] segments = axonDbClient.getClosedEventSegments();
return stream(segments)
.map(File::new)
.filter(File::exists)
.map(File::getName)
.filter(name -> name.endsWith(".events"))
.map(name -> config.getEventsBackupPath() + name)
.map(File::new)
.filter(f -> !f.exists())
.peek(f -> log.info("Found new segment {}", f.getName()))
.count() > 0;
}
private void backupEvents() {
Stopwatch stopwatch = createStarted();
String[] segments = axonDbClient.getClosedEventSegments();
stream(segments)
.map(File::new)
.filter(File::exists)
.forEach(f -> fileService.copy(f, config.getEventsBackupPath()));
log.info("Copied event segments in {}", stopwatch);
}
private void backupSnapshots() {
Stopwatch stopwatch = createStarted();
String[] segments = axonDbClient.getClosedSnapshotsSegments();
stream(segments)
.map(File::new)
.filter(File::exists)
.forEach(f -> fileService.copy(f, config.getSnapshotsBackupPath()));
log.info("Copied snapshot segments in {}", stopwatch);
}
}
spring:
application:
name: axondb-backup-client
server:
port: 9080
management:
endpoints:
web:
base-path: /manage
exposure:
include: "*"
backup:
fixedDelayMs: 60000
initialDelayMs: 60000
axondbHost: http://127.0.0.1:8023/axondb
eventsUrl: /v1/backup/filenames?type=Event
snapshotsUrl: /v1/backup/filenames?type=Snapshot
dbDumpUrl: /v1/backup/createControlDbBackup
axondbToken: ${AXON_DB_TOKEN:"dummy-token"}
eventsBackupPath: "/eventsBackup/"
snapshotsBackupPath: "/snapshotsBackup/"
dbDumpBackupPath: "/dbDumpBackup/"
\ No newline at end of file
package nl.trifork.axondbbackupclient;
import io.axoniq.axondb.client.AxonDBConfiguration;
import io.axoniq.axondb.client.axon.AxonDBEventStore;
import org.axonframework.commandhandling.TargetAggregateIdentifier;
import org.axonframework.config.Configuration;
import org.axonframework.eventhandling.EventBus;
import org.axonframework.eventhandling.GenericEventMessage;
import org.axonframework.serialization.xml.XStreamSerializer;
import org.junit.Ignore;
import org.junit.Test;
import java.util.stream.LongStream;
import static io.axoniq.axondb.client.AxonDBConfiguration.newBuilder;
import static java.util.UUID.randomUUID;
import static org.axonframework.config.DefaultConfigurer.defaultConfiguration;
@Ignore("for manual usage")
public class AxonDbLoadTest {
EventBus eventBus = eventBus();
@Test
public void should_publish_events() {
LongStream.range(0, 300_000L)
.parallel()
.mapToObj(i -> new SomeEvent())
.map(GenericEventMessage::asEventMessage)
.forEach(eventBus::publish);
}
private AxonDBEventStore eventBus() {
AxonDBConfiguration axonDbConfiguration = newBuilder("localhost:8123").build();
AxonDBEventStore axonDbEventStore = new AxonDBEventStore(axonDbConfiguration, new XStreamSerializer());
Configuration config = defaultConfiguration().configureEventBus(c -> axonDbEventStore).buildConfiguration();
config.start();
return axonDbEventStore;
}
private static class SomeEvent {
@TargetAggregateIdentifier String targetId = randomUUID().toString();
}
}
package nl.trifork.axondbbackupclient;
import nl.trifork.axondbbackupclient.axondb_client.AxonDbRestClient;
import nl.trifork.axondbbackupclient.backup_services.ControlDBBackupService;
import nl.trifork.axondbbackupclient.backup_services.FileIoService;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class ControlDBBackupServiceTest {
@Rule
public TemporaryFolder tmp = new TemporaryFolder();
BackupConfig config = new BackupConfig();
AxonDbRestClient client = mock(AxonDbRestClient.class);
FileIoService fileService = mock(FileIoService.class);
ControlDBBackupService subj = new ControlDBBackupService(config, client, fileService);
@Test
public void should_create_and_copy_db_dump() throws IOException {
File dumpFile = tmp.newFile("control1234.db");
File dbBackup = tmp.newFolder("dbBackup");
when(client.createControlDbDump()).thenReturn(dumpFile.getAbsolutePath());
config.setDbDumpBackupPath(dbBackup.getAbsolutePath());
//when
subj.backup();
//then
verify(fileService).copy(dumpFile, config.getDbDumpBackupPath());
assertThat(dumpFile.exists()).isFalse();
}
}
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment