Add template for assignment 3

This commit is contained in:
2021-05-17 15:44:23 +02:00
parent 1363e83aa3
commit be629aa432
64 changed files with 3556 additions and 0 deletions
@@ -0,0 +1,38 @@
package dst.ass3.messaging;
/**
* Contains several constants related to the RabbitMQ infrastructure and expected names for queues, exchanges and
* routing keys.
*/
public final class Constants {
public static final String RMQ_HOST = "192.168.99.99";
public static final String RMQ_PORT = "5672";
public static final String RMQ_VHOST = "/";
public static final String RMQ_USER = "dst";
public static final String RMQ_PASSWORD = "dst";
public static final String RMQ_API_PORT = "15672";
public static final String RMQ_API_URL = "http://" + RMQ_HOST + ":" + RMQ_API_PORT + "/api/";
public static final String QUEUE_AT_VIENNA = "dst.at_vienna";
public static final String QUEUE_AT_LINZ = "dst.at_linz";
public static final String QUEUE_DE_BERLIN = "dst.de_berlin";
public static final String[] WORK_QUEUES = {
QUEUE_AT_VIENNA,
QUEUE_AT_LINZ,
QUEUE_DE_BERLIN
};
public static final String TOPIC_EXCHANGE = "dst.workers";
public static final String ROUTING_KEY_AT_VIENNA = "requests.at_vienna";
public static final String ROUTING_KEY_AT_LINZ = "requests.at_linz";
public static final String ROUTING_KEY_DE_BERLIN = "requests.de_berlin";
private Constants() {
// util class
}
}
@@ -0,0 +1,54 @@
package dst.ass3.messaging;
import java.util.Objects;
public class GeoPoint {
private Double longitude;
private Double latitude;
public GeoPoint() {
}
public GeoPoint(Double longitude, Double latitude) {
this.longitude = longitude;
this.latitude = latitude;
}
public Double getLongitude() {
return longitude;
}
public void setLongitude(Double longitude) {
this.longitude = longitude;
}
public Double getLatitude() {
return latitude;
}
public void setLatitude(Double latitude) {
this.latitude = latitude;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
GeoPoint geoPoint = (GeoPoint) o;
return Objects.equals(getLongitude(), geoPoint.getLongitude()) &&
Objects.equals(getLatitude(), geoPoint.getLatitude());
}
@Override
public int hashCode() {
return Objects.hash(getLongitude(), getLatitude());
}
}
@@ -0,0 +1,21 @@
package dst.ass3.messaging;
import java.io.Closeable;
import java.io.IOException;
public interface IMessagingFactory extends Closeable {
IQueueManager createQueueManager();
IRequestGateway createRequestGateway();
IWorkloadMonitor createWorkloadMonitor();
/**
* Closes any resource the factory may create.
*
* @throws IOException propagated exceptions
*/
@Override
void close() throws IOException;
}
@@ -0,0 +1,28 @@
package dst.ass3.messaging;
import java.io.Closeable;
import java.io.IOException;
/**
* Responsible for creating and tearing down all necessary RabbitMQ queues or exchanges necessary for running the system.
*/
public interface IQueueManager extends Closeable {
/**
* Initializes all queues or topic exchanges necessary for running the system.
*/
void setUp();
/**
* Removes all queues or topic exchanged associated with the system.
*/
void tearDown();
/**
* Closes underlying conection or resources, if any.
*
* @throws IOException propagated exceptions
*/
@Override
void close() throws IOException;
}
@@ -0,0 +1,21 @@
package dst.ass3.messaging;
import java.io.Closeable;
import java.io.IOException;
public interface IRequestGateway extends Closeable {
/**
* Serializes and routes a request to the correct queue.
* @param request the request
*/
void submitRequest(TripRequest request);
/**
* Closes any resources that may have been initialized (connections, channels, etc.)
*
* @throws IOException propagated exceptions
*/
@Override
void close() throws IOException;
}
@@ -0,0 +1,34 @@
package dst.ass3.messaging;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
public interface IWorkloadMonitor extends Closeable {
/**
* Returns for each region the amount of waiting requests.
*
* @return a map
*/
Map<Region, Long> getRequestCount();
/**
* Returns the amount of workers for each region. This can be deduced from the amount of consumers to each
* queue.
*
* @return a map
*/
Map<Region, Long> getWorkerCount();
/**
* Returns for each region the average processing time of the last 10 recorded requests. The data comes from
* subscriptions to the respective topics.
*
* @return a map
*/
Map<Region, Double> getAverageProcessingTime();
@Override
void close() throws IOException;
}
@@ -0,0 +1,7 @@
package dst.ass3.messaging;
public enum Region {
AT_VIENNA,
AT_LINZ,
DE_BERLIN
}
@@ -0,0 +1,71 @@
package dst.ass3.messaging;
import java.util.Objects;
public class TripRequest {
private String id;
private Region region;
private GeoPoint pickup;
public TripRequest() {
}
public TripRequest(String id, Region region, GeoPoint pickup) {
this.id = id;
this.region = region;
this.pickup = pickup;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Region getRegion() {
return region;
}
public void setRegion(Region region) {
this.region = region;
}
public GeoPoint getPickup() {
return pickup;
}
public void setPickup(GeoPoint pickup) {
this.pickup = pickup;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
TripRequest that = (TripRequest) o;
return Objects.equals(getId(), that.getId()) &&
getRegion() == that.getRegion() &&
Objects.equals(getPickup(), that.getPickup());
}
@Override
public int hashCode() {
return Objects.hash(getId(), getRegion(), getPickup());
}
@Override
public String toString() {
return "TripRequest{" +
"id='" + id + '\'' +
", region=" + region +
", pickup=" + pickup +
'}';
}
}
@@ -0,0 +1,76 @@
package dst.ass3.messaging;
import java.util.Objects;
/**
* Message sent by a worker after it is finished processing a request.
*/
public class WorkerResponse {
/**
* The ID of the original {@link TripRequest}.
*/
private String requestId;
/**
* The time it took to process the request (in milliseconds).
*/
private Long processingTime;
/**
* The proposed driver
*/
private Long driverId;
public String getRequestId() {
return requestId;
}
public void setRequestId(String requestId) {
this.requestId = requestId;
}
public Long getProcessingTime() {
return processingTime;
}
public void setProcessingTime(Long processingTime) {
this.processingTime = processingTime;
}
public Long getDriverId() {
return driverId;
}
public void setDriverId(Long driverId) {
this.driverId = driverId;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
WorkerResponse that = (WorkerResponse) o;
return Objects.equals(getRequestId(), that.getRequestId()) &&
Objects.equals(getProcessingTime(), that.getProcessingTime()) &&
Objects.equals(getDriverId(), that.getDriverId());
}
@Override
public int hashCode() {
return Objects.hash(getRequestId(), getProcessingTime(), getDriverId());
}
@Override
public String toString() {
return "WorkerResponse{" +
"requestId='" + requestId + '\'' +
", processingTime=" + processingTime +
", driverId=" + driverId +
'}';
}
}
@@ -0,0 +1,32 @@
package dst.ass3.messaging.impl;
import dst.ass3.messaging.IMessagingFactory;
import dst.ass3.messaging.IQueueManager;
import dst.ass3.messaging.IRequestGateway;
import dst.ass3.messaging.IWorkloadMonitor;
public class MessagingFactory implements IMessagingFactory {
@Override
public IQueueManager createQueueManager() {
// TODO
return null;
}
@Override
public IRequestGateway createRequestGateway() {
// TODO
return null;
}
@Override
public IWorkloadMonitor createWorkloadMonitor() {
// TODO
return null;
}
@Override
public void close() {
// implement if needed
}
}
@@ -0,0 +1,16 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} - %highlight(%5p) [%12.12thread] %cyan(%-40.40logger{39}): %m%n</pattern>
</encoder>
</appender>
<logger name="org.apache.http" level="WARN"/>
<root level="${log.level:-INFO}">
<appender-ref ref="STDOUT"/>
</root>
</configuration>
@@ -0,0 +1,51 @@
package dst.ass3.messaging;
import com.rabbitmq.http.client.Client;
import org.junit.rules.ExternalResource;
import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
import org.springframework.amqp.rabbit.core.RabbitAdmin;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.apache.http.auth.Credentials;
public class RabbitResource extends ExternalResource {
private RabbitAdmin admin;
private Client manager;
private CachingConnectionFactory connectionFactory;
private RabbitTemplate client;
@Override
protected void before() throws Throwable {
manager = new Client(Constants.RMQ_API_URL, Constants.RMQ_USER, Constants.RMQ_PASSWORD);
connectionFactory = new CachingConnectionFactory(Constants.RMQ_HOST);
connectionFactory.setUsername(Constants.RMQ_USER);
connectionFactory.setPassword(Constants.RMQ_PASSWORD);
client = new RabbitTemplate(connectionFactory);
admin = new RabbitAdmin(connectionFactory);
}
@Override
protected void after() {
connectionFactory.destroy();
}
public Client getManager() {
return manager;
}
public RabbitTemplate getClient() {
return client;
}
public RabbitAdmin getAdmin() {
return admin;
}
public CachingConnectionFactory getConnectionFactory() {
return connectionFactory;
}
}
@@ -0,0 +1,13 @@
package dst.ass3.messaging.impl;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
QueueManagerTest.class,
RequestGatewayTest.class,
WorkloadMonitorTest.class
})
public class Ass3_1_Suite {
}
@@ -0,0 +1,97 @@
package dst.ass3.messaging.impl;
import com.rabbitmq.http.client.domain.ExchangeInfo;
import com.rabbitmq.http.client.domain.QueueInfo;
import dst.ass3.messaging.IMessagingFactory;
import dst.ass3.messaging.IQueueManager;
import dst.ass3.messaging.RabbitResource;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.springframework.amqp.core.Exchange;
import org.springframework.amqp.core.Queue;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static dst.ass3.messaging.Constants.*;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
public class QueueManagerTest {
@Rule
public RabbitResource rabbit = new RabbitResource();
@Rule
public Timeout timeout = new Timeout(10, TimeUnit.SECONDS);
private IMessagingFactory factory = new MessagingFactory();
private IQueueManager queueManager;
@Before
public void setUp() throws Exception {
factory = new MessagingFactory();
queueManager = factory.createQueueManager();
}
@After
public void tearDown() throws Exception {
try {
queueManager.close();
} catch (IOException e) {
// ignore
}
}
@Test
public void setUp_createsQueues() throws Exception {
queueManager.setUp();
try {
List<QueueInfo> queues = rabbit.getManager().getQueues();
assertThat(queues.size(), not(0));
// make sure all work queues exist
List<String> queueNames = queues.stream().map(QueueInfo::getName).collect(Collectors.toList());
Arrays.stream(WORK_QUEUES)
.forEach(wq -> assertThat(queueNames, hasItem(wq)));
} finally {
queueManager.tearDown();
}
}
@Test
public void setUp_createsExchange() throws Exception {
queueManager.setUp();
try {
ExchangeInfo exchange = rabbit.getManager().getExchange(RMQ_VHOST, TOPIC_EXCHANGE);
assertThat(exchange, notNullValue());
} finally {
queueManager.tearDown();
}
}
@Test
public void tearDown_removesQueues() throws Exception {
queueManager.setUp();
queueManager.tearDown();
List<QueueInfo> queues = rabbit.getManager().getQueues();
List<String> queueNames = queues.stream().map(QueueInfo::getName).collect(Collectors.toList());
Arrays.stream(WORK_QUEUES)
.forEach(wq -> assertThat(queueNames, not(hasItem(wq))));
}
@Test
public void tearDown_removesExchange() throws Exception {
queueManager.setUp();
queueManager.tearDown();
ExchangeInfo exchange = rabbit.getManager().getExchange(RMQ_VHOST, TOPIC_EXCHANGE);
assertThat(exchange, nullValue());
}
}
@@ -0,0 +1,112 @@
package dst.ass3.messaging.impl;
import dst.ass3.messaging.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.Message;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
public class RequestGatewayTest {
private static final Logger LOG = LoggerFactory.getLogger(RequestGatewayTest.class);
@Rule
public RabbitResource rabbit = new RabbitResource();
@Rule
public Timeout timeout = new Timeout(10, TimeUnit.SECONDS);
private IMessagingFactory factory;
private IQueueManager queueManager;
private IRequestGateway requestGateway;
@Before
public void setUp() throws Exception {
factory = new MessagingFactory();
queueManager = factory.createQueueManager();
requestGateway = factory.createRequestGateway();
queueManager.setUp();
}
@After
public void tearDown() throws Exception {
queueManager.tearDown();
requestGateway.close();
queueManager.close();
factory.close();
}
@Test
public void submitRequest_routesRequestsToCorrectQueues() throws Exception {
TripRequest r1 = new TripRequest("id1", Region.AT_VIENNA, new GeoPoint(34.22, -22.11));
TripRequest r2 = new TripRequest("id2", Region.AT_VIENNA, new GeoPoint(98.2, -1.11));
TripRequest r3 = new TripRequest("id3", Region.AT_LINZ, new GeoPoint(85.33, -25d));
LOG.info("Sending request {}", r1);
requestGateway.submitRequest(r1);
LOG.info("Sending request {}", r2);
requestGateway.submitRequest(r2);
LOG.info("Sending request {}", r3);
requestGateway.submitRequest(r3);
LOG.info("Taking requests from queue {}", Constants.QUEUE_AT_VIENNA);
Message m1 = rabbit.getClient().receive(Constants.QUEUE_AT_VIENNA, 1000);
assertThat(m1, notNullValue());
LOG.info("Taking requests from queue {}", Constants.QUEUE_AT_VIENNA);
Message m2 = rabbit.getClient().receive(Constants.QUEUE_AT_VIENNA, 1000);
assertThat(m2, notNullValue());
LOG.info("Taking requests from queue {}", Constants.QUEUE_AT_LINZ);
Message m3 = rabbit.getClient().receive(Constants.QUEUE_AT_LINZ, 1000);
assertThat(m3, notNullValue());
assertThat("Expected queue to be empty as no request for that region were issued",
rabbit.getClient().receive(Constants.QUEUE_DE_BERLIN, 1000), nullValue());
}
@Test
public void submitRequest_serializesIntoJsonFormat() throws Exception {
TripRequest r1 = new TripRequest("id1", Region.AT_VIENNA, new GeoPoint(1d, 2d));
TripRequest r2 = new TripRequest("id2", Region.AT_LINZ, new GeoPoint(3d, 4d));
TripRequest r3 = new TripRequest("id3", Region.DE_BERLIN, new GeoPoint(5d, 6d));
LOG.info("Sending request {}", r1);
requestGateway.submitRequest(r1);
LOG.info("Sending request {}", r2);
requestGateway.submitRequest(r2);
LOG.info("Sending request {}", r3);
requestGateway.submitRequest(r3);
LOG.info("Taking request from queue {}", Constants.QUEUE_AT_VIENNA);
Message m1 = rabbit.getClient().receive(Constants.QUEUE_AT_VIENNA, 1000);
assertThat(m1, notNullValue());
assertThat(new String(m1.getBody()),
equalTo("{\"id\":\"id1\",\"region\":\"AT_VIENNA\",\"pickup\":{\"longitude\":1.0,\"latitude\":2.0}}"));
LOG.info("Taking request from queue {}", Constants.QUEUE_AT_LINZ);
Message m2 = rabbit.getClient().receive(Constants.QUEUE_AT_LINZ, 1000);
assertThat(m2, notNullValue());
assertThat(new String(m2.getBody()),
equalTo("{\"id\":\"id2\",\"region\":\"AT_LINZ\",\"pickup\":{\"longitude\":3.0,\"latitude\":4.0}}"));
LOG.info("Taking request from queue {}", Constants.QUEUE_DE_BERLIN);
Message m3 = rabbit.getClient().receive(Constants.QUEUE_DE_BERLIN, 1000);
assertThat(m3, notNullValue());
assertThat(new String(m3.getBody()),
equalTo("{\"id\":\"id3\",\"region\":\"DE_BERLIN\",\"pickup\":{\"longitude\":5.0,\"latitude\":6.0}}"));
}
}
@@ -0,0 +1,169 @@
package dst.ass3.messaging.impl;
import com.rabbitmq.http.client.domain.QueueInfo;
import dst.ass3.messaging.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.rabbit.listener.MessageListenerContainer;
import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer;
import org.springframework.context.Lifecycle;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import static dst.ass3.messaging.Constants.TOPIC_EXCHANGE;
import static dst.ass3.messaging.Constants.WORK_QUEUES;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.MatcherAssert.assertThat;
public class WorkloadMonitorTest {
private static final Logger LOG = LoggerFactory.getLogger(RequestGatewayTest.class);
@Rule
public RabbitResource rabbit = new RabbitResource();
@Rule
public Timeout timeout = new Timeout(60, TimeUnit.SECONDS);
private IMessagingFactory factory;
private IQueueManager queueManager;
private IRequestGateway requestGateway;
private IWorkloadMonitor workloadMonitor;
@Before
public void setUp() throws Exception {
factory = new MessagingFactory();
queueManager = factory.createQueueManager();
requestGateway = factory.createRequestGateway();
queueManager.setUp();
workloadMonitor = factory.createWorkloadMonitor();
}
@After
public void tearDown() throws Exception {
queueManager.tearDown();
requestGateway.close();
queueManager.close();
factory.close();
}
@Test
public void getRequestCount_returnsCorrectCount() throws Exception {
try {
Map<Region, Long> countForRegion = new HashMap<>();
for (Region region : Region.values()) {
countForRegion.put(region, ThreadLocalRandom.current().nextLong(10, 20 + 1));
for (long i = 0; i < countForRegion.get(region); i++) {
GeoPoint pickup = new GeoPoint(48.11, 16.22);
TripRequest request = new TripRequest("id" + i, region, pickup);
LOG.info("Sending request {}", request);
requestGateway.submitRequest(request);
}
}
// wait for the messages to be processed by rabbit
Thread.sleep(2000);
assertThat(workloadMonitor.getRequestCount(), equalTo(countForRegion));
} finally {
workloadMonitor.close();
}
}
@Test
public void multipleWorkloadMonitors_uniqueQueueForEachMonitor() throws Exception {
try (IWorkloadMonitor workloadMonitor2 = factory.createWorkloadMonitor();
IWorkloadMonitor workloadMonitor3 = factory.createWorkloadMonitor();) {
long nonWorkQueues = rabbit.getManager().getQueues().stream().filter(q -> !Arrays.asList(WORK_QUEUES).contains(q.getName())).count();
assertThat(nonWorkQueues, greaterThanOrEqualTo(3L));
} finally {
workloadMonitor.close();
}
}
@Test
public void getAverageProcessingTime_correctAverageTime() throws Exception {
try {
Map<Region, Double> avgTimes = new HashMap<>();
for (Region region : Region.values()) {
long count = ThreadLocalRandom.current().nextLong(15, 25);
long regionTime = 0;
for (long i = 0; i < count; i++) {
long requestTime = ThreadLocalRandom.current().nextLong(1000, 20000 + 1);
if (count - i <= 10) {
regionTime += requestTime;
}
String body = String.format("{\"requestId\": \"%s\", \"processingTime\": \"%d\", \"driverId\": \"1\"}", UUID.randomUUID(), requestTime);
LOG.info("Sending request {}", body);
rabbit.getClient().convertAndSend(TOPIC_EXCHANGE, "requests." + region.toString().toLowerCase(), body);
}
avgTimes.put(region, ((double) regionTime / 10));
}
// wait for the messages to be processed by rabbit
Thread.sleep(2000);
assertThat(workloadMonitor.getAverageProcessingTime(), equalTo(avgTimes));
} finally {
workloadMonitor.close();
}
}
@Test
public void getWorkerCount_returnsCorrectCount() throws Exception {
try {
// spawn a random number of consumers
Map<Region, Collection<MessageListenerContainer>> consumersForRegion = new HashMap<>();
Map<Region, Long> consumerCountForRegion = new HashMap<>();
for (Region region : Region.values()) {
List<MessageListenerContainer> consumers = new ArrayList<>();
consumersForRegion.put(region, consumers);
consumerCountForRegion.put(region, ThreadLocalRandom.current().nextLong(10, 20 + 1));
for (long i = 0; i < consumerCountForRegion.get(region); i++) {
consumers.add(spawnConsumer("dst." + region.toString().toLowerCase()));
}
}
// wait for rabbit to get to know the new consumers
Thread.sleep(2000);
Map<Region, Long> workerCount = workloadMonitor.getWorkerCount();
// stop all consumers
consumersForRegion.entrySet().stream().map(Map.Entry::getValue).flatMap(Collection::stream).forEach(Lifecycle::stop);
assertThat(workerCount, equalTo(consumerCountForRegion));
} finally {
workloadMonitor.close();
}
}
private MessageListenerContainer spawnConsumer(String queue) {
SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(rabbit.getConnectionFactory());
container.addQueueNames(queue);
container.start();
return container;
}
@Test
public void close_removesQueues() throws Exception {
workloadMonitor.close();
List<QueueInfo> queues = rabbit.getManager().getQueues();
long nonWorkQueues = rabbit.getManager().getQueues().stream().filter(q -> !Arrays.asList(WORK_QUEUES).contains(q.getName())).count();
assertThat(nonWorkQueues, is(0L));
}
}