Add template for assignment 3

This commit is contained in:
2021-05-17 15:44:23 +02:00
parent 1363e83aa3
commit be629aa432
64 changed files with 3556 additions and 0 deletions
@@ -0,0 +1,15 @@
package dst.ass3.event;
/**
* Constants.
*/
public final class Constants {
/**
* The TCP port of the {@link EventPublisher}.
*/
public static final int EVENT_PUBLISHER_PORT = 1338;
private Constants() {
}
}
@@ -0,0 +1,16 @@
package dst.ass3.event;
/**
* Creates your {@link IEventProcessingEnvironment} and {@link IEventSourceFunction} implementation instances.
*/
public class EventProcessingFactory {
public static IEventProcessingEnvironment createEventProcessingEnvironment() {
// TODO
return null;
}
public static IEventSourceFunction createEventSourceFunction() {
// TODO
return null;
}
}
@@ -0,0 +1,173 @@
package dst.ass3.event;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import dst.ass3.event.model.domain.ITripEventInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.shaded.netty4.io.netty.bootstrap.ServerBootstrap;
import org.apache.flink.shaded.netty4.io.netty.channel.ChannelFuture;
import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInitializer;
import org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup;
import org.apache.flink.shaded.netty4.io.netty.channel.group.ChannelGroup;
import org.apache.flink.shaded.netty4.io.netty.channel.group.DefaultChannelGroup;
import org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.flink.shaded.netty4.io.netty.channel.socket.SocketChannel;
import org.apache.flink.shaded.netty4.io.netty.channel.socket.nio.NioServerSocketChannel;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.serialization.ClassResolvers;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.serialization.ObjectDecoder;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.serialization.ObjectEncoder;
/**
* An EventPublisher accepts incoming TCP socket connections on a given port and is able to broadcast {@link ITripEventInfo}
* objects to these clients.
*/
public class EventPublisher {
private static final Logger LOG = LoggerFactory.getLogger(EventPublisher.class);
private final Object clientChannelMonitor = new Object();
private final int port;
private final AtomicBoolean closed;
private EventLoopGroup bossGroup;
private EventLoopGroup workerGroup;
private ChannelGroup clientChannels;
public EventPublisher(int port) {
this.port = port;
this.closed = new AtomicBoolean(false);
}
public int getPort() {
return port;
}
/**
* Broadcast an event to all listening channels. Does nothing if no clients are connected.
*
* @param event the event to publish
* @throws IllegalStateException if the publisher hasn't been started yet or has been closed
*/
public void publish(ITripEventInfo event) {
if (clientChannels == null || closed.get()) {
throw new IllegalStateException();
}
clientChannels.writeAndFlush(event).syncUninterruptibly();
// wait a bit for event to propagate
try {
Thread.sleep(10);
} catch (InterruptedException e) {}
}
/**
* Like {@link #publish(ITripEventInfo)} but waits for a given number of milliseconds and then passes the current system
* time to a factory function.
*
* @param delay the delay in ms
* @param provider the provider
*/
public void publish(long delay, Function<Long, ITripEventInfo> provider) {
if (delay > 0) {
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
}
}
publish(provider.apply(System.currentTimeMillis()));
}
/**
* This method blocks if no clients are connected, and is notified as soon as a client connects. If clients are
* connected, the method returns immediately.
*/
public void waitForClients() {
if (clientChannels.isEmpty()) {
LOG.debug("Waiting for clients to connect...");
synchronized (clientChannelMonitor) {
try {
clientChannelMonitor.wait();
} catch (InterruptedException e) {
LOG.debug("Interrupted while waiting on client connections", e);
}
}
}
}
public int getConnectedClientCount() {
if (clientChannels == null || closed.get()) {
throw new IllegalStateException();
}
return clientChannels.size();
}
/**
* Closes all active client connections.
*/
public void dropClients() {
if (clientChannels == null || closed.get()) {
throw new IllegalStateException();
}
clientChannels.close().syncUninterruptibly().group().clear();
}
/**
* Start the server and accept incoming connections. Will call {@link #close()} if an error occurs during
* connection.
*/
public void start() {
bossGroup = new NioEventLoopGroup();
workerGroup = new NioEventLoopGroup();
clientChannels = new DefaultChannelGroup(workerGroup.next());
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ClientChannelInitializer());
// Bind and start to accept incoming connections
ChannelFuture f = b.bind(port).addListener(future -> {
if (!future.isSuccess()) {
LOG.error("Error while binding socket");
close();
}
}).syncUninterruptibly();
LOG.info("Accepting connections on {}", f.channel());
}
/**
* Closes all channels and resources.
*/
public void close() {
if (closed.compareAndSet(false, true)) {
LOG.info("Shutting down event loops");
if (bossGroup != null) {
bossGroup.shutdownGracefully();
}
if (workerGroup != null) {
workerGroup.shutdownGracefully();
}
}
}
private class ClientChannelInitializer extends ChannelInitializer<SocketChannel> {
@Override
public void initChannel(SocketChannel ch) throws Exception {
LOG.info("Initializing client channel {}", ch);
clientChannels.add(ch);
ch.pipeline()
.addFirst(new ObjectEncoder())
.addFirst(new ObjectDecoder(ClassResolvers.cacheDisabled(ClassLoader.getSystemClassLoader())));
synchronized (clientChannelMonitor) {
clientChannelMonitor.notifyAll();
}
}
}
}
@@ -0,0 +1,178 @@
package dst.ass3.event;
import java.lang.reflect.Proxy;
import java.net.SocketAddress;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import dst.ass3.event.model.domain.ITripEventInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.shaded.netty4.io.netty.bootstrap.Bootstrap;
import org.apache.flink.shaded.netty4.io.netty.channel.Channel;
import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandlerContext;
import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandlerAdapter;
import org.apache.flink.shaded.netty4.io.netty.channel.ChannelOption;
import org.apache.flink.shaded.netty4.io.netty.channel.EventLoopGroup;
import org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoopGroup;
import org.apache.flink.shaded.netty4.io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.serialization.ClassResolvers;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.serialization.ObjectDecoder;
import org.apache.flink.shaded.netty4.io.netty.handler.codec.serialization.ObjectEncoder;
import org.apache.flink.shaded.netty4.io.netty.util.concurrent.Future;
/**
* An EventSubscriber receives ITripEventInfo objects through a netty SocketChannel. Create and connect an
* EventSubscriber using {@link #subscribe(SocketAddress)}. To receive events, call {@link #receive()}.
*/
public class EventSubscriber {
private static final Logger LOG = LoggerFactory.getLogger(EventSubscriber.class);
private static final ITripEventInfo POISON_PILL = (ITripEventInfo) Proxy.newProxyInstance(
ITripEventInfo.class.getClassLoader(), new Class[]{ITripEventInfo.class}, (p, m, a) -> null);
private final SocketAddress publisherAddress;
private final BlockingQueue<ITripEventInfo> queue;
private volatile boolean closed;
private Channel channel;
private EventLoopGroup loop;
private EventSubscriber(SocketAddress publisherAddress) {
this.publisherAddress = publisherAddress;
this.queue = new LinkedBlockingQueue<>();
}
/**
* Blocks to receive the next ITripEventInfo published into the channel. Returns {@code null} if the underlying
* channel has been closed or the thread was interrupted.
*
* @return the next ITripEventInfo object
* @throws IllegalStateException thrown if the previous call returned null and the channel was closed
*/
public ITripEventInfo receive() throws IllegalStateException {
synchronized (queue) {
if (closed && queue.isEmpty()) {
throw new IllegalStateException();
}
}
ITripEventInfo event;
try {
event = queue.take();
if (event == POISON_PILL) {
return null;
} else {
return event;
}
} catch (InterruptedException e) {
return null;
}
}
private Future<?> start() {
loop = new NioEventLoopGroup();
channel = new Bootstrap()
.group(loop)
.channel(NioSocketChannel.class)
.option(ChannelOption.TCP_NODELAY, true)
.handler(new EventSubscriberHandler())
.connect(publisherAddress) // ChannelFuture
.addListener(future -> {
if (!future.isSuccess()) {
LOG.error("Error while connecting");
close();
}
})
.syncUninterruptibly()
.channel();
LOG.info("Connected to channel {}", channel);
return loop.submit(() -> {
try {
channel.closeFuture().sync();
} catch (InterruptedException e) {
// noop
} finally {
close();
}
});
}
/**
* Closes all resources and threads used by the EventSubscriber.
*/
public void close() {
try {
if (loop != null) {
synchronized (queue) {
if (!loop.isShutdown() && !loop.isTerminated() && !loop.isShuttingDown()) {
LOG.info("Shutting down event loop");
loop.shutdownGracefully();
}
}
}
} finally {
synchronized (queue) {
if (!closed) {
LOG.debug("Adding poison pill to queue");
closed = true;
queue.add(POISON_PILL);
}
}
}
}
/**
* Creates a new EventSubscriber that connects to given SocketAddress.
*
* @param address the socket address
* @return a new EventSubscriber
*/
public static EventSubscriber subscribe(SocketAddress address) {
EventSubscriber eventSubscriber = new EventSubscriber(address);
eventSubscriber.start();
return eventSubscriber;
}
private class EventSubscriberHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
ctx.read();
if (!(msg instanceof ITripEventInfo)) {
LOG.error("Unknown message type received {}", msg);
return;
}
synchronized (queue) {
if (!closed) {
queue.add((ITripEventInfo) msg);
}
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
LOG.error("EventSubscriberHandler caught an exception", cause);
ctx.close();
close();
}
@Override
public void channelActive(ChannelHandlerContext ctx) {
ctx.pipeline()
.addFirst(new ObjectEncoder())
.addFirst(new ObjectDecoder(ClassResolvers.cacheDisabled(ClassLoader.getSystemClassLoader())));
}
}
}
@@ -0,0 +1,39 @@
package dst.ass3.event;
import dst.ass3.event.model.events.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.windowing.time.Time;
/**
* This class should be used to implement the event processing steps as described in the assignment. The test classes
* will inject SinkFunctions, create a new StreamExecutionEnvironment and then call {@link
* #initialize(StreamExecutionEnvironment)}.
*/
public interface IEventProcessingEnvironment {
/**
* Initializes the event processing graph on the {@link StreamExecutionEnvironment}. This function is called
* after all sinks have been set.
*/
void initialize(StreamExecutionEnvironment env);
/**
* Sets the timeout limit of a streaming event.
*
* @param time the timeout limit
*/
void setMatchingDurationTimeout(Time time);
void setLifecycleEventStreamSink(SinkFunction<LifecycleEvent> sink);
void setMatchingDurationStreamSink(SinkFunction<MatchingDuration> sink);
void setAverageMatchingDurationStreamSink(SinkFunction<AverageMatchingDuration> sink);
void setMatchingTimeoutWarningStreamSink(SinkFunction<MatchingTimeoutWarning> sink);
void setTripFailedWarningStreamSink(SinkFunction<TripFailedWarning> sink);
void setAlertStreamSink(SinkFunction<Alert> sink);
}
@@ -0,0 +1,25 @@
package dst.ass3.event;
import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import dst.ass3.event.model.domain.ITripEventInfo;
/**
* A RichFunction & SourceFunction for ITripEventInfo objects.
*/
public interface IEventSourceFunction extends RichFunction, SourceFunction<ITripEventInfo> {
@Override
void open(Configuration parameters) throws Exception;
@Override
void close() throws Exception;
@Override
void run(SourceContext<ITripEventInfo> ctx) throws Exception;
@Override
void cancel();
}
@@ -0,0 +1,54 @@
package dst.ass3.event.dto;
import java.io.Serializable;
import dst.ass3.event.model.domain.ITripEventInfo;
import dst.ass3.event.model.domain.Region;
import dst.ass3.event.model.domain.TripState;
public class TripEventInfoDTO implements Serializable, ITripEventInfo {
private static final long serialVersionUID = 4134104076758220138L;
private Long tripId;
private Long timestamp;
private TripState state;
private Region region;
public TripEventInfoDTO(Long tripId, Long timestamp, TripState state, Region region) {
this.tripId = tripId;
this.timestamp = timestamp;
this.state = state;
this.region = region;
}
@Override
public Long getTripId() {
return tripId;
}
@Override
public Long getTimestamp() {
return timestamp;
}
@Override
public TripState getState() {
return state;
}
@Override
public Region getRegion() {
return region;
}
@Override
public String toString() {
return "TripEventInfoDTO{" +
"tripId=" + tripId +
", timestamp=" + timestamp +
", state=" + state +
", region=" + region +
'}';
}
}
@@ -0,0 +1,9 @@
package dst.ass3.event.model.domain;
public interface ITripEventInfo {
Long getTripId();
Long getTimestamp();
TripState getState();
Region getRegion();
}
@@ -0,0 +1,7 @@
package dst.ass3.event.model.domain;
public enum Region {
AT_VIENNA,
AT_LINZ,
DE_BERLIN
}
@@ -0,0 +1,5 @@
package dst.ass3.event.model.domain;
public enum TripState {
CREATED, QUEUED, MATCHED, APPROACHING, IN_PROGRESS, CANCELLED, COMPLETED
}
@@ -0,0 +1,49 @@
package dst.ass3.event.model.events;
import java.io.Serializable;
import java.util.List;
import dst.ass3.event.model.domain.Region;
/**
* A system alert that aggregates several warnings.
*/
public class Alert implements Serializable {
private static final long serialVersionUID = -4561132671849230635L;
private Region region;
private List<Warning> warnings;
public Alert() {
}
public Alert(Region region, List<Warning> warnings) {
this.region = region;
this.warnings = warnings;
}
public Region getRegion() {
return region;
}
public void setRegion(Region region) {
this.region = region;
}
public List<Warning> getWarnings() {
return warnings;
}
public void setWarnings(List<Warning> warnings) {
this.warnings = warnings;
}
@Override
public String toString() {
return "Alert{" +
"region='" + region + '\'' +
", warnings=" + warnings +
'}';
}
}
@@ -0,0 +1,48 @@
package dst.ass3.event.model.events;
import java.io.Serializable;
import dst.ass3.event.model.domain.Region;
/**
* The average of several {@link MatchingDuration} values.
*/
public class AverageMatchingDuration implements Serializable {
private static final long serialVersionUID = -3767582104941550250L;
private Region region;
private double duration;
public AverageMatchingDuration() {
}
public AverageMatchingDuration(Region region, double duration) {
this.region = region;
this.duration = duration;
}
public Region getRegion() {
return region;
}
public void setRegion(Region region) {
this.region = region;
}
public double getDuration() {
return duration;
}
public void setDuration(double duration) {
this.duration = duration;
}
@Override
public String toString() {
return "AverageMatchingDuration{" +
"region='" + region + '\'' +
", duration=" + duration +
'}';
}
}
@@ -0,0 +1,84 @@
package dst.ass3.event.model.events;
import java.io.Serializable;
import dst.ass3.event.model.domain.ITripEventInfo;
import dst.ass3.event.model.domain.Region;
import dst.ass3.event.model.domain.TripState;
/**
* Indicates a change in the lifecycle state of an ITripEventInfo.
*/
public class LifecycleEvent implements Serializable {
private static final long serialVersionUID = 8665269919851487210L;
/**
* The id of the trip, as returned by {@link ITripEventInfo#getTripId()}.
*/
private long tripId;
private TripState state;
private Region region;
/**
* The instant the event was recorded (unix epoch in milliseconds)
*/
private long timestamp;
public LifecycleEvent() {
}
public LifecycleEvent(ITripEventInfo eventInfo) {
this(eventInfo.getTripId(), eventInfo.getState(), eventInfo.getRegion(), eventInfo.getTimestamp());
}
public LifecycleEvent(long tripId, TripState state, Region region, long timestamp) {
this.tripId = tripId;
this.state = state;
this.region = region;
this.timestamp = timestamp;
}
public long getTripId() {
return tripId;
}
public void setTripId(long tripId) {
this.tripId = tripId;
}
public TripState getState() {
return state;
}
public void setState(TripState state) {
this.state = state;
}
public Region getRegion() {
return region;
}
public void setRegion(Region region) {
this.region = region;
}
public long getTimestamp() {
return timestamp;
}
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
@Override
public String toString() {
return "LifecycleEvent{" +
"tripId=" + tripId +
", state=" + state +
", region=" + region +
", timestamp=" + timestamp +
'}';
}
}
@@ -0,0 +1,59 @@
package dst.ass3.event.model.events;
import java.io.Serializable;
import dst.ass3.event.model.domain.Region;
/**
* Indicates the amount of time an ITripEventInfo took to get from CREATED to MATCHED.
*/
public class MatchingDuration implements Serializable {
private static final long serialVersionUID = -6976972381929291369L;
private long eventId;
private Region region;
private long duration;
public MatchingDuration() {
}
public MatchingDuration(long eventId, Region region, long duration) {
this.eventId = eventId;
this.region = region;
this.duration = duration;
}
public long getEventId() {
return eventId;
}
public void setEventId(long eventId) {
this.eventId = eventId;
}
public Region getRegion() {
return region;
}
public void setRegion(Region region) {
this.region = region;
}
public long getDuration() {
return duration;
}
public void setDuration(long duration) {
this.duration = duration;
}
@Override
public String toString() {
return "MatchingDuration{" +
"eventId=" + eventId +
", region='" + region + '\'' +
", duration=" + duration +
'}';
}
}
@@ -0,0 +1,37 @@
package dst.ass3.event.model.events;
import dst.ass3.event.model.domain.Region;
/**
* Warning that indicates that a matching event has not reached the lifecycle state MATCHED within a given time frame.
*/
public class MatchingTimeoutWarning extends Warning {
private static final long serialVersionUID = 7955599732178947649L;
private long tripId;
public MatchingTimeoutWarning() {
super(null);
}
public MatchingTimeoutWarning(long tripId, Region region) {
super(region);
this.tripId = tripId;
}
public long getTripId() {
return tripId;
}
public void setTripId(long tripId) {
this.tripId = tripId;
}
@Override
public String toString() {
return "MatchingTimeoutWarning{" +
"tripId=" + tripId +
'}';
}
}
@@ -0,0 +1,37 @@
package dst.ass3.event.model.events;
import dst.ass3.event.model.domain.Region;
/**
* Indicates that matching a trip has probably failed.
*/
public class TripFailedWarning extends Warning {
private static final long serialVersionUID = -9120187311385112769L;
private long eventId;
public TripFailedWarning() {
super(null);
}
public TripFailedWarning(long eventId, Region region) {
super(region);
this.eventId = eventId;
}
public long getEventId() {
return eventId;
}
public void setEventId(long eventId) {
this.eventId = eventId;
}
@Override
public String toString() {
return "TripFailedWarning{" +
"eventId=" + eventId +
'}';
}
}
@@ -0,0 +1,37 @@
package dst.ass3.event.model.events;
import java.io.Serializable;
import dst.ass3.event.model.domain.Region;
/**
* Base class for region warnings.
*/
public abstract class Warning implements Serializable {
private static final long serialVersionUID = 273266717303711974L;
private Region region;
public Warning() {
}
public Warning(Region region) {
this.region = region;
}
public Region getRegion() {
return region;
}
public void setRegion(Region region) {
this.region = region;
}
@Override
public String toString() {
return "Warning{" +
"region='" + region + '\'' +
'}';
}
}
@@ -0,0 +1 @@
// TODO: add the data from the execution plan export
+16
View File
@@ -0,0 +1,16 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} - %highlight(%5p) [%12.12thread] %cyan(%-40.40logger{39}): %m%n</pattern>
</encoder>
</appender>
<logger name="org.apache.flink" level="WARN"/> <!-- change this to INFO if you want to see more flink logging data-->
<root level="${log.level:-INFO}">
<appender-ref ref="STDOUT"/>
</root>
</configuration>
@@ -0,0 +1,69 @@
package dst.ass3.event;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.flink.shaded.guava18.com.google.common.util.concurrent.MoreExecutors;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Ass3EventTestBase.
*/
public abstract class Ass3EventTestBase {
private static final Logger LOG = LoggerFactory.getLogger(Ass3EventTestBase.class);
protected EventPublisher publisher;
protected StreamExecutionEnvironment flink;
protected ExecutorService executor;
private static EventPublisher previousPublisher;
@Before
public void setUpResources() throws Exception {
executor = Executors.newCachedThreadPool();
if (previousPublisher != null) {
previousPublisher.close();
}
publisher = createEventPublisher();
previousPublisher = publisher;
publisher.start();
flink = createStreamExecutionEnvironment();
}
@After
public void tearDownResources() throws Exception {
publisher.close();
MoreExecutors.shutdownAndAwaitTermination(executor, 5, TimeUnit.SECONDS);
previousPublisher = null;
}
protected EventPublisher createEventPublisher() {
return new EventPublisher(Constants.EVENT_PUBLISHER_PORT);
}
protected StreamExecutionEnvironment createStreamExecutionEnvironment() {
return StreamExecutionEnvironment.createLocalEnvironment(1);
}
protected static void sleep(long ms) {
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
// ignore
}
}
protected static long now() {
return System.currentTimeMillis();
}
}
@@ -0,0 +1,23 @@
package dst.ass3.event;
import dst.ass3.event.tests.Ass3_3_2Test;
import dst.ass3.event.tests.Ass3_3_3Test;
import dst.ass3.event.tests.Ass3_3_4Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import dst.ass3.event.tests.Ass3_3_1Test;
/**
* Ass3EventTestSuite.
*/
@RunWith(Suite.class)
@SuiteClasses({
Ass3_3_1Test.class,
Ass3_3_2Test.class,
Ass3_3_3Test.class,
Ass3_3_4Test.class
})
public class Ass3EventTestSuite {
}
@@ -0,0 +1,98 @@
package dst.ass3.event;
import static org.junit.Assert.assertNotNull;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import dst.ass3.event.model.events.*;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* EventProcessingTestBase.
*/
public class EventProcessingTestBase extends Ass3EventTestBase {
private static final Logger LOG = LoggerFactory.getLogger(EventProcessingTestBase.class);
@Rule
public Timeout timeout = new Timeout(15, TimeUnit.SECONDS);
private IEventProcessingEnvironment epe;
protected StaticQueueSink<LifecycleEvent> lifecycleEvents;
protected StaticQueueSink<MatchingDuration> matchingDurations;
protected StaticQueueSink<AverageMatchingDuration> averageMatchingDurations;
protected StaticQueueSink<MatchingTimeoutWarning> matchingTimeoutWarnings;
protected StaticQueueSink<TripFailedWarning> tripFailedWarnings;
protected StaticQueueSink<Alert> alerts;
@Before
public void setUpEnvironment() throws Exception {
epe = EventProcessingFactory.createEventProcessingEnvironment();
assertNotNull("#createEventProcessingEnvironment() not implemented", epe);
lifecycleEvents = new StaticQueueSink<>("lifecycleEvents");
matchingDurations = new StaticQueueSink<>("matchingDurations");
averageMatchingDurations = new StaticQueueSink<>("averageMatchingDurations");
matchingTimeoutWarnings = new StaticQueueSink<>("matchingTimeoutWarnings");
tripFailedWarnings = new StaticQueueSink<>("tripFailedWarnings");
alerts = new StaticQueueSink<>("alerts");
epe.setLifecycleEventStreamSink(lifecycleEvents);
epe.setMatchingDurationStreamSink(matchingDurations);
epe.setAverageMatchingDurationStreamSink(averageMatchingDurations);
epe.setMatchingTimeoutWarningStreamSink(matchingTimeoutWarnings);
epe.setTripFailedWarningStreamSink(tripFailedWarnings);
epe.setAlertStreamSink(alerts);
epe.setMatchingDurationTimeout(Time.seconds(15));
}
public JobExecutionResult initAndExecute() throws Exception {
return initAndExecute(null);
}
public JobExecutionResult initAndExecute(Consumer<IEventProcessingEnvironment> initializer) throws Exception {
try {
if (initializer != null) {
initializer.accept(epe);
}
LOG.info("Initializing StreamExecutionEnvironment with {}", epe);
epe.initialize(flink);
} catch (Exception e) {
LOG.error("Error while initializing StreamExecutionEnvironment", e);
throw e;
}
try {
LOG.info("Executing flink {}", flink);
return flink.execute();
} catch (Exception e) {
LOG.error("Error while executing flink", e);
throw e;
}
}
public Future<JobExecutionResult> initAndExecuteAsync(Consumer<IEventProcessingEnvironment> initializer) {
return executor.submit(() -> initAndExecute(initializer));
}
public Future<JobExecutionResult> initAndExecuteAsync() {
return executor.submit(() -> initAndExecute());
}
@After
public void tearDownCollectors() throws Exception {
StaticQueueSink.clearAll();
}
}
@@ -0,0 +1,83 @@
package dst.ass3.event;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
/**
* A SinkFunction that collects objects into a queue located in a shared global state. Each collector accesses a
* specific key in the shared state.
*
* @param <T> the sink input type
*/
public class StaticQueueSink<T> implements SinkFunction<T> {
private static final long serialVersionUID = -3965500756295835669L;
private static Map<String, BlockingQueue<?>> state = new ConcurrentHashMap<>();
private String key;
public StaticQueueSink(String key) {
this.key = key;
}
@Override
public void invoke(T value, Context context) throws Exception {
get().add(value);
}
public void clear() {
get().clear();
}
public List<T> take(int n) {
List<T> list = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
try {
list.add(get().take());
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted while accessing queue", e);
}
}
return list;
}
public T take() {
try {
return get().take();
} catch (InterruptedException e) {
return null;
}
}
public T poll(long ms) {
try {
return get().poll(ms, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
return null;
}
}
public synchronized BlockingQueue<T> get() {
return get(key);
}
@SuppressWarnings("unchecked")
private static <R> BlockingQueue<R> get(String key) {
return (BlockingQueue) state.computeIfAbsent(key, k -> new LinkedBlockingQueue<>());
}
public static void clearAll() {
state.clear();
}
}
@@ -0,0 +1,152 @@
package dst.ass3.event.tests;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.NotSerializableException;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import dst.ass3.event.dto.TripEventInfoDTO;
import dst.ass3.event.model.domain.ITripEventInfo;
import dst.ass3.event.model.domain.Region;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import dst.ass3.event.Ass3EventTestBase;
import dst.ass3.event.EventProcessingFactory;
import dst.ass3.event.IEventSourceFunction;
import dst.ass3.event.model.domain.TripState;
public class Ass3_3_1Test extends Ass3EventTestBase {
private static final Logger LOG = LoggerFactory.getLogger(Ass3_3_1Test.class);
@Rule
public Timeout timeout = new Timeout(15, TimeUnit.SECONDS);
private IEventSourceFunction sourceFunction;
@Before
public void setUp() throws Exception {
sourceFunction = EventProcessingFactory.createEventSourceFunction();
assertNotNull("EventProcessingFactory#createEventSourceFunction() not implemented", sourceFunction);
}
@Test
public void open_shouldConnectToSubscriber() throws Exception {
assertEquals(
"IEventSourceFunction should not be connected upon construction",
0, publisher.getConnectedClientCount()
);
sourceFunction.open(new Configuration());
publisher.waitForClients();
assertEquals(
"Expected IEventSourceFunction to connect to publisher after open is called",
1, publisher.getConnectedClientCount()
);
}
@Test
public void run_shouldCollectPublishedEvents() throws Exception {
sourceFunction.open(new Configuration());
publisher.waitForClients();
Future<List<ITripEventInfo>> result = executor.submit(() -> {
MockContext<ITripEventInfo> ctx = new MockContext<>();
LOG.info("Running IEventSourceFunction with MockContext");
sourceFunction.run(ctx);
LOG.info("Done running IEventSourceFunction, returning collected events");
return ctx.collected;
});
publisher.publish(new TripEventInfoDTO(1L, 0L, TripState.CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(2L, 0L, TripState.CREATED, Region.DE_BERLIN));
sleep(1000);
LOG.info("Calling cancel on SourceFunction");
sourceFunction.cancel();
LOG.info("Dropping subscriber connections");
publisher.dropClients();
LOG.info("Calling close on SourceFunction");
sourceFunction.close();
List<ITripEventInfo> collected = result.get();
assertEquals(2, collected.size());
ITripEventInfo e0 = collected.get(0);
ITripEventInfo e1 = collected.get(1);
assertEquals(1L, e0.getTripId(), 0);
assertEquals(2L, e1.getTripId(), 0);
}
@Test
public void shouldBeSerializable() throws Exception {
try (ObjectOutputStream out = new ObjectOutputStream(new ByteArrayOutputStream())) {
out.writeObject(sourceFunction);
out.flush();
} catch (NotSerializableException e) {
fail("Implementation of IEventSourceFunction is not serializable");
}
}
private static class MockContext<T> implements SourceFunction.SourceContext<T> {
private final Object checkpointLock = new Object();
private List<T> collected = new ArrayList<>();
public List<T> getCollected() {
return collected;
}
@Override
public void collect(T element) {
collected.add(element);
}
@Override
public void collectWithTimestamp(T element, long timestamp) {
collected.add(element);
}
@Override
public void emitWatermark(Watermark mark) {
}
@Override
public void markAsTemporarilyIdle() {
}
@Override
public Object getCheckpointLock() {
return checkpointLock;
}
@Override
public void close() {
}
}
}
@@ -0,0 +1,89 @@
package dst.ass3.event.tests;
import dst.ass3.event.EventProcessingTestBase;
import dst.ass3.event.dto.TripEventInfoDTO;
import dst.ass3.event.model.domain.Region;
import dst.ass3.event.model.events.LifecycleEvent;
import org.apache.flink.api.common.JobExecutionResult;
import org.hamcrest.Matchers;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.Future;
import static dst.ass3.event.model.domain.TripState.CREATED;
import static dst.ass3.event.model.domain.TripState.MATCHED;
import static dst.ass3.event.model.domain.TripState.QUEUED;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.hamcrest.MatcherAssert.assertThat;
public class Ass3_3_2Test extends EventProcessingTestBase {
private static final Logger LOG = LoggerFactory.getLogger(Ass3_3_2Test.class);
@Test
public void lifecycleEventStream_worksCorrectly() throws Exception {
// run flink in new thread
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync();
// wait until a subscriber connects
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
LifecycleEvent event;
long then = System.currentTimeMillis();
// causes LifecycleEvent 0
LOG.info("Publishing e0");
publisher.publish(new TripEventInfoDTO(0L, then, CREATED, Region.AT_VIENNA));
LOG.info("Collecting LifecycleEvent for e0");
event = lifecycleEvents.take();
LOG.info("Round-trip took {}ms", System.currentTimeMillis() - then);
assertEquals("Event ID not set correctly", 0L, event.getTripId());
assertEquals("State not set correctly", CREATED, event.getState());
assertEquals("Region not set correctly", Region.AT_VIENNA, event.getRegion());
assertThat("Timestamp not set correctly", event.getTimestamp(), Matchers.greaterThanOrEqualTo(then));
assertThat("Timestamp not set correctly", event.getTimestamp(),
Matchers.lessThanOrEqualTo(System.currentTimeMillis()));
// should be filtered
LOG.info("Publishing e1, should be filtered");
publisher.publish(new TripEventInfoDTO(1L, now(), CREATED, null));
assertNull("Events without a region should be filtered", lifecycleEvents.poll(500));
// causes LifecycleEvent 1
LOG.info("Publishing e2");
publisher.publish(new TripEventInfoDTO(2L, now(), QUEUED, Region.DE_BERLIN));
LOG.info("Collecting LifecycleEvent for e2");
event = lifecycleEvents.take();
assertEquals(2L, event.getTripId());
assertEquals(QUEUED, event.getState());
// should be filtered
LOG.info("Publishing e3, should be filtered");
publisher.publish(new TripEventInfoDTO(3L, now(), CREATED, null));
assertNull("Events without a region should be filtered", lifecycleEvents.poll(500));
// causes LifecycleEvent 2
LOG.info("Publishing e4");
publisher.publish(new TripEventInfoDTO(4L, now(), MATCHED, Region.AT_LINZ));
LOG.info("Collecting LifecycleEvent for e4");
event = lifecycleEvents.take();
assertEquals(4L, event.getTripId());
assertEquals(MATCHED, event.getState());
// disconnect subscribers
publisher.dropClients();
// wait for execution to end
flinkExecution.get();
}
}
@@ -0,0 +1,211 @@
package dst.ass3.event.tests;
import dst.ass3.event.EventProcessingTestBase;
import dst.ass3.event.dto.TripEventInfoDTO;
import dst.ass3.event.model.domain.Region;
import dst.ass3.event.model.events.LifecycleEvent;
import dst.ass3.event.model.events.MatchingDuration;
import dst.ass3.event.model.events.MatchingTimeoutWarning;
import dst.ass3.event.model.events.TripFailedWarning;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Future;
import static dst.ass3.event.model.domain.TripState.*;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.*;
public class Ass3_3_3Test extends EventProcessingTestBase {
private static final Logger LOG = LoggerFactory.getLogger(Ass3_3_3Test.class);
@Test
public void matchingDurations_areCalculatedCorrectly() throws Exception {
// tests whether duration calculation and stream keying works correctly
// expects LifecycleEvent stream to work correctly
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync();
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
LifecycleEvent e1Start;
LifecycleEvent e2Start;
LifecycleEvent e1End;
LifecycleEvent e2End;
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, CREATED, Region.AT_VIENNA)); // 1 starts before 2
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, CREATED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(3L, t, CREATED, Region.AT_VIENNA)); // 3 never finishes
LOG.info("Waiting for LifecycleEvent for event 1 being CREATED");
e1Start = lifecycleEvents.take();
LOG.info("Waiting for LifecycleEvent for event 2 being CREATED");
e2Start = lifecycleEvents.take();
LOG.info("Waiting for LifecycleEvent for event 3 being CREATED");
lifecycleEvents.take();
sleep(500);
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, QUEUED, Region.AT_VIENNA));
LOG.info("Waiting for LifecycleEvent for event 1 being QUEUED");
lifecycleEvents.take();
LOG.info("Waiting for LifecycleEvent for event 2 being QUEUED");
lifecycleEvents.take();
sleep(500); // event 2 took about ~1000ms
publisher.publish(new TripEventInfoDTO(2L, now(), MATCHED, Region.AT_VIENNA)); // 2 finishes before 1
LOG.info("Waiting for LifecycleEvent for event 2 being MATCHED");
e2End = lifecycleEvents.take();
sleep(500); // event 1 took about ~1500ms
publisher.publish(new TripEventInfoDTO(1L, now(), MATCHED, Region.AT_VIENNA));
LOG.info("Waiting for LifecycleEvent for event 1 being MATCHED");
e1End = lifecycleEvents.take();
LOG.info("Collecting MatchingDuration event for event 2");
MatchingDuration d0 = matchingDurations.take();
LOG.info("Collecting MatchingDuration event for event 1");
MatchingDuration d1 = matchingDurations.take();
assertEquals("Expected event 2 to finish first", 2L, d0.getEventId()); // event 2 finished before 1
assertEquals("Expected event 1 to finish last", 1L, d1.getEventId());
assertThat("Expected MatchingDuration to be >= 0", d0.getDuration(), greaterThan(0L));
assertThat("Expected MatchingDuration to be >= 0", d1.getDuration(), greaterThan(0L));
assertEquals("MatchingDuration was not calculated from LifecycleEvents correctly",
e2End.getTimestamp() - e2Start.getTimestamp(), d0.getDuration(), 100);
assertEquals("MatchingDuration was not calculated from LifecycleEvents correctly",
e1End.getTimestamp() - e1Start.getTimestamp(), d1.getDuration(), 100);
publisher.dropClients();
flinkExecution.get();
}
@Test
public void durationsWithInterleavedEvents_areCalculatedCorrectly() throws Exception {
// tests whether CEP rule is tolerant towards multiple state changes between QUEUED and MATCHED
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync(e ->
e.setMatchingDurationTimeout(Time.seconds(1))
);
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, CREATED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, CREATED, Region.AT_VIENNA)); // never finishes
// change state several times (tests another aspect of the CEP rule)
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.dropClients();
flinkExecution.get();
List<MatchingDuration> result = new ArrayList<>(matchingDurations.get());
assertEquals("Expected one event to have finished", 1, result.size());
MatchingDuration d0 = result.get(0);
assertEquals("Expected event 1 to have finished", 1L, d0.getEventId());
}
@Test
public void timeoutWarnings_areEmittedCorrectly() throws Exception {
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync(e -> {
e.setMatchingDurationTimeout(Time.seconds(1));
});
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
publisher.publish(new TripEventInfoDTO(1L, now(), CREATED, Region.AT_VIENNA)); // never finishes
sleep(100);
publisher.publish(new TripEventInfoDTO(2L, now(), CREATED, Region.AT_VIENNA)); // never finishes
// confounding event
sleep(100);
publisher.publish(new TripEventInfoDTO(1L, now(), QUEUED, Region.AT_VIENNA));
sleep(1500); // wait for event to time out
publisher.dropClients();
LOG.info("Waiting for Flink execution to end");
flinkExecution.get();
LOG.info("Collecting timeout warning for event 1");
MatchingTimeoutWarning w1 = matchingTimeoutWarnings.take();
LOG.info("Collecting timeout warning for event 2");
MatchingTimeoutWarning w2 = matchingTimeoutWarnings.take();
assertEquals("Expected event 1 to time out first", 1L, w1.getTripId());
assertEquals("Expected event 2 to time out second", 2L, w2.getTripId());
}
@Test
public void tripFailedWarnings_areEmittedCorrectly() throws Exception {
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync();
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
publisher.publish(0, t -> new TripEventInfoDTO(1L, t, CREATED, Region.AT_VIENNA));
publisher.publish(0, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(0, t -> new TripEventInfoDTO(2L, t, CREATED, Region.AT_VIENNA));
publisher.publish(0, t -> new TripEventInfoDTO(2L, t, QUEUED, Region.AT_VIENNA));
// event 1 fail #1
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
// event 2 fail #1
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, QUEUED, Region.AT_VIENNA));
// event 1 fail #2
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
// event 2 fail #2 and then success
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, MATCHED, Region.AT_VIENNA));
LOG.info("Checking that no TripFailedWarning was issued yet");
assertNull(tripFailedWarnings.poll(500));
LOG.info("Triggering third failure");
// event 1 fail #3
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
LOG.info("Waiting for TripFailedWarning for event 1");
TripFailedWarning warning = tripFailedWarnings.take();
assertEquals(1L, warning.getEventId());
assertEquals(Region.AT_VIENNA, warning.getRegion());
publisher.dropClients();
flinkExecution.get();
}
}
@@ -0,0 +1,308 @@
package dst.ass3.event.tests;
import dst.ass3.event.EventProcessingTestBase;
import dst.ass3.event.dto.TripEventInfoDTO;
import dst.ass3.event.model.domain.Region;
import dst.ass3.event.model.events.*;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import static dst.ass3.event.model.domain.TripState.*;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.*;
public class Ass3_3_4Test extends EventProcessingTestBase {
private static final Logger LOG = LoggerFactory.getLogger(Ass3_3_4Test.class);
@Test
public void multipleTripFailures_triggerAlert() throws Exception {
// checks that the window works correctly
// expects TripFailedWarning stream to work correctly
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync();
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
Consumer<Long> causeWarning = (eventId) -> {
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, QUEUED, Region.AT_VIENNA));
};
// all of these events will fail
publisher.publish(new TripEventInfoDTO(1L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(1L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(2L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(2L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(3L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(3L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(4L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(4L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(5L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(5L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(6L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(6L, now(), QUEUED, Region.AT_VIENNA));
// warning 3 warnings
causeWarning.accept(1L);
causeWarning.accept(2L);
causeWarning.accept(3L);
LOG.info("Collecting alert for first three warnings");
assertNotNull(alerts.take());
LOG.info("Checking that the fourth warning does not trigger an alert");
causeWarning.accept(4L);
assertNull(alerts.poll(500));
LOG.info("Checking that the fifth warning does not trigger an alert");
causeWarning.accept(5L);
assertNull(alerts.poll(500));
LOG.info("Checking that the sixth warning triggered an alert");
causeWarning.accept(6L);
assertNotNull(alerts.take());
publisher.dropClients();
flinkExecution.get();
}
@Test
public void matchingFailuresAndTimeouts_triggerAlert() throws Exception {
// checks whether keying works correctly, and whether Warning streams are unioned correctly
// expects both warning streams to work correctly
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync(e ->
e.setMatchingDurationTimeout(Time.seconds(3))
);
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
BiConsumer<Long, Region> causeWarning = (eventId, region) -> {
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, MATCHED, region));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, QUEUED, region));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, MATCHED, region));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, QUEUED, region));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, MATCHED, region));
publisher.publish(5, t -> new TripEventInfoDTO(eventId, t, QUEUED, region));
};
publisher.publish(new TripEventInfoDTO(1L, now(), CREATED, Region.AT_VIENNA)); // vienna e1 will fail
publisher.publish(new TripEventInfoDTO(1L, now(), QUEUED, Region.AT_VIENNA)); // vienna e1 will fail
publisher.publish(new TripEventInfoDTO(2L, now(), CREATED, Region.AT_VIENNA)); // vienna e2 will fail
publisher.publish(new TripEventInfoDTO(2L, now(), QUEUED, Region.AT_VIENNA)); // vienna e2 will fail
publisher.publish(new TripEventInfoDTO(3L, now(), CREATED, Region.AT_VIENNA)); // vienna e3 will time out
publisher.publish(new TripEventInfoDTO(3L, now(), QUEUED, Region.AT_VIENNA)); // vienna e3 will time out
publisher.publish(new TripEventInfoDTO(4L, now(), CREATED, Region.DE_BERLIN)); // berlin e4 will fail
publisher.publish(new TripEventInfoDTO(4L, now(), QUEUED, Region.DE_BERLIN)); // berlin e4 will fail
// s1 warning #1
causeWarning.accept(1L, Region.AT_VIENNA);
// s1 warning #2
causeWarning.accept(2L, Region.AT_VIENNA);
// s2 warning #1
causeWarning.accept(4L, Region.DE_BERLIN);
LOG.info("Checking that no alert has been issued yet");
assertNull(alerts.poll(500));
// make sure the other events don't time out
publisher.publish(new TripEventInfoDTO(1L, now(), MATCHED, Region.AT_VIENNA)); // vienna e1 will fail
publisher.publish(new TripEventInfoDTO(2L, now(), MATCHED, Region.AT_VIENNA)); // vienna e2 will fail
publisher.publish(new TripEventInfoDTO(4L, now(), MATCHED, Region.DE_BERLIN)); // berlin e4 will fail
sleep(4000); // waiting for e3 to time out
publisher.dropClients();
flinkExecution.get();
LOG.info("Collecting Alert event");
Alert alert = alerts.take();
assertEquals("Expected only a single alert", 0, alerts.get().size());
assertEquals(Region.AT_VIENNA, alert.getRegion());
assertEquals("An alert should comprise three warnings", 3, alert.getWarnings().size());
Warning w0 = alert.getWarnings().get(0);
Warning w1 = alert.getWarnings().get(1);
Warning w2 = alert.getWarnings().get(2);
assertThat(w0, instanceOf(TripFailedWarning.class));
assertThat(w1, instanceOf(TripFailedWarning.class));
assertThat(w2, instanceOf(MatchingTimeoutWarning.class));
assertEquals(Region.AT_VIENNA, w0.getRegion());
assertEquals(Region.AT_VIENNA, w1.getRegion());
assertEquals(Region.AT_VIENNA, w2.getRegion());
}
@Test
public void averageMatchingDurationWindow_worksCorrectly() throws Exception {
// makes sure the event is triggered at the correct instant
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync();
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
sleep(250);
publisher.publish(new TripEventInfoDTO(1L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(2L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(3L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(4L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(5L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(6L, now(), CREATED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(7L, now(), CREATED, Region.DE_BERLIN));
sleep(100);
publisher.publish(new TripEventInfoDTO(1L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(2L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(3L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(4L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(5L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(6L, now(), QUEUED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(7L, now(), QUEUED, Region.DE_BERLIN));
sleep(100);
publisher.publish(new TripEventInfoDTO(1L, now(), MATCHED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(2L, now(), MATCHED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(3L, now(), MATCHED, Region.AT_VIENNA));
publisher.publish(new TripEventInfoDTO(4L, now(), MATCHED, Region.AT_VIENNA));
// the first four events should not trigger anything
LOG.info("Checking AverageMatchingDuration events after the first four MATCHED events of AT_VIENNA");
assertNull(averageMatchingDurations.poll(500));
// this event is from a different region
publisher.publish(new TripEventInfoDTO(7L, now(), MATCHED, Region.DE_BERLIN));
LOG.info("Checking AverageMatchingDuration events after the first MATCHED event of DE_BERLIN");
assertNull(averageMatchingDurations.poll(500));
// fifth event in s1 triggers the window operation
publisher.publish(new TripEventInfoDTO(5L, now(), MATCHED, Region.AT_VIENNA));
LOG.info("Collecting AverageMatchingDuration event for AT_VIENNA");
AverageMatchingDuration event = averageMatchingDurations.take();
assertNotNull(event);
assertEquals(Region.AT_VIENNA, event.getRegion());
// should be in a new window and therefore not trigger
publisher.publish(new TripEventInfoDTO(6L, now(), MATCHED, Region.AT_VIENNA));
LOG.info("Checking AverageMatchingDuration events after the sixth MATCHED event of AT_VIENNA");
assertNull(averageMatchingDurations.poll(500));
publisher.dropClients();
flinkExecution.get();
}
@Test
public void averageMatchingDurations_areCalculatedCorrectly() throws Exception {
// makes sure the keying works properly and that the calculation is done from MatchingDuration events
// requires MatchingDuration events to be calculated correctly
Future<JobExecutionResult> flinkExecution = initAndExecuteAsync();
LOG.info("Waiting for subscribers to connect");
publisher.waitForClients();
List<MatchingDuration> viennaDurations = new ArrayList<>(5);
List<MatchingDuration> berlinDurations = new ArrayList<>(5);
sleep(250);
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, CREATED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, CREATED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(3L, t, CREATED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(4L, t, CREATED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(5L, t, CREATED, Region.AT_VIENNA));
sleep(250);
publisher.publish(5, t -> new TripEventInfoDTO(6L, t, CREATED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(7L, t, CREATED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(8L, t, CREATED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(9L, t, CREATED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(10L, t, CREATED, Region.DE_BERLIN));
sleep(125);
publisher.publish(5, t -> new TripEventInfoDTO(6L, t, QUEUED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(7L, t, QUEUED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(8L, t, QUEUED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(9L, t, QUEUED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(10L, t, QUEUED, Region.DE_BERLIN));
sleep(125);
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(3L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(4L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(5L, t, QUEUED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(6L, t, MATCHED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(7L, t, MATCHED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(8L, t, MATCHED, Region.DE_BERLIN));
LOG.info("Collecting MatchingDuration events 1,2,3 for DE_BERLIN");
berlinDurations.addAll(matchingDurations.take(3));
sleep(500);
publisher.publish(5, t -> new TripEventInfoDTO(1L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(2L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(3L, t, MATCHED, Region.AT_VIENNA));
LOG.info("Collecting MatchingDuration events 1,2,3 for AT_VIENNA");
viennaDurations.addAll(matchingDurations.take(3));
publisher.publish(5, t -> new TripEventInfoDTO(9L, t, MATCHED, Region.DE_BERLIN));
publisher.publish(5, t -> new TripEventInfoDTO(10L, t, MATCHED, Region.DE_BERLIN));
LOG.info("Collecting MatchingDuration events 4,5 for DE_BERLIN");
berlinDurations.addAll(matchingDurations.take(2));
sleep(500);
publisher.publish(5, t -> new TripEventInfoDTO(4L, t, MATCHED, Region.AT_VIENNA));
publisher.publish(5, t -> new TripEventInfoDTO(5L, t, MATCHED, Region.AT_VIENNA));
LOG.info("Collecting MatchingDuration events 4,5 for AT_VIENNA");
viennaDurations.addAll(matchingDurations.take(2));
LOG.info("Collecting AverageMatchingDuration event for DE_BERLIN");
AverageMatchingDuration e0 = averageMatchingDurations.take(); // berlin
LOG.info("Collecting AverageMatchingDuration event for AT_VIENNA");
AverageMatchingDuration e1 = averageMatchingDurations.take(); // vienna
assertEquals("Expected calculation for berlin to have been triggered first", e0.getRegion(), Region.DE_BERLIN);
assertEquals("Expected calculation for vienna to have been triggered second", e1.getRegion(), Region.AT_VIENNA);
assertEquals("Wrong number of MatchingDuration events for AT_VIENNA", 5, viennaDurations.size());
assertEquals("Wrong number of MatchingDuration events for DE_BERLIN", 5, berlinDurations.size());
double viennaAvg = viennaDurations.stream().mapToLong(MatchingDuration::getDuration).average().orElse(-1);
double berlinAvg = berlinDurations.stream().mapToLong(MatchingDuration::getDuration).average().orElse(-1);
assertEquals("Average duration was not calculated from MatchingDuration events correctly", e0.getDuration(),
berlinAvg, 100);
assertEquals("Average duration was not calculated from MatchingDuration events correctly", e1.getDuration(),
viennaAvg, 100);
publisher.dropClients();
flinkExecution.get();
}
}
+16
View File
@@ -0,0 +1,16 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} - %highlight(%5p) [%12.12thread] %cyan(%-40.40logger{39}): %m%n</pattern>
</encoder>
</appender>
<logger name="org.apache.flink.shaded" level="WARN"/>
<root level="${log.level:-INFO}">
<appender-ref ref="STDOUT"/>
</root>
</configuration>