|
Kafka是一种高吞吐量的分布式发布订阅消息系统，有如下特性：
相关术语介绍
在这里我们用了一个第三方库叫Confluent.kafka，在nuget上搜索一下就出来了，感谢原作者。 新建一个 .net core类库项目安装第三方依赖库，如下图所示： 新建一个SUPKafkaTopicConsumer类这是用来创建并初始化消费者，接下来看看这个类里面包含了什么。
public delegate void OnReceivedHandle(object data); 初始化消费者，构造函数中传入kafka地址，以及要订阅的组groupId，另外注入了log4net记录日志信息。 public class SUPKafkaTopicConsumer<TKey, TValue>
{
private IConsumer<TKey, TValue> consumer;
private SUPLogger logger_;
private string BootStrapServer;
private string GroupId;
public SUPKafkaTopicConsumer(string bootStrapServer, string groupId, SUPLogger logger = null)
{
BootStrapServer = bootStrapServer;
GroupId = groupId;
logger_ = logger;
}
public bool Init()
{
try
{
var conf = new ConsumerConfig
{
GroupId = GroupId,
BootstrapServers = BootStrapServer,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false // 设置非自动偏移，业务逻辑完成后手动处理偏移，防止数据丢失
};
consumer = new ConsumerBuilder<TKey, TValue>(conf)
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build();
return true;
}
catch (Exception ex)
{
throw;
}
}
public event OnReceivedHandle onReceivedHandle;
public void Subscribe(string topic, bool isCommit)
{
try
{
if (consumer != null)
{
consumer.Subscribe(topic);
while (true)
{
var consume = consumer.Consume();
if (onReceivedHandle != null)
{
onReceivedHandle(consume);
if (isCommit)
{
consumer.Commit(consume);
}
}
}
}
}
catch (Exception ex)
{
//consumer.Close();
throw ex;
}
}
public void UnSubscribe()
{
if (consumer != null)
{
consumer.Unsubscribe();
}
}新建生产者类
public interface ISUPKafkaProducer<Tkey,TValue>
{
ISendResult Send(Tkey key, TValue value, string topic,Action<DeliveryReport<Tkey, TValue>> sendCallBack = null);
ISendResult Send(Tkey key, TValue value, TopicPartition topicPartition, Action<DeliveryReport<Tkey, TValue>> sendCallBack = null);
ISendResult AsyncSend(Tkey key, TValue value,string topic);
ISendResult AsyncSend(Tkey key, TValue value, TopicPartition topicPartition);
}
internal class SUPKafkaTopicProducer<Tkey, TValue> : ISUPKafkaProducer<Tkey, TValue>
{
private IProducer<Tkey, TValue> producer;
private SUPLogger logger_;
private string m_bootStrapServer;
public SUPKafkaTopicProducer(string bootStrapServer,SUPLogger logger = null)
{
m_bootStrapServer = bootStrapServer;
logger_ = logger;
}
public bool Init()
{
try
{
var config = new ProducerConfig
{
BootstrapServers = m_bootStrapServer
};
producer = new ProducerBuilder<Tkey, TValue>(config)
.SetErrorHandler((producer, error) =>
{
logger_.Fatal(string.Format("Kafka Error Handler {0},ErrorCode:{2},Reason:{3}",
m_bootStrapServer, error.Code, error.Reason));
})
.SetLogHandler((producer, msg) =>
{
logger_.Info(string.Format("Kafka Log Handler {0}-{1},Name:{2},Message:{3}",
m_bootStrapServer, msg.Name, msg.Message));
})
.Build();
return true;
}
catch (Exception ex)
{
throw ex;
}
}实现继承至ISUPKafkaProducer<Tkey, TValue>的方法 public ISendResult Send(Tkey key, TValue value,string topic, Action<DeliveryReport<Tkey, TValue>> sendCallBack = null)
{
try
{
if (producer != null)
{
var message = new Message<Tkey, TValue>
{
Value = value,
Key = key
};
producer.Produce(topic, message, sendCallBack);
return new SendResult(true);
}
else
{
return new SendResult(true, "没有初始化生产者");
}
}
catch (Exception ex)
{
throw ex;
}
}
public ISendResult Send(Tkey key, TValue value, TopicPartition topicPartition, Action<DeliveryReport<Tkey, TValue>> sendCallBack = null)
{
try
{
if (producer != null)
{
var message = new Message<Tkey, TValue>
{
Value = value,
Key = key
};
producer.Produce(topicPartition, message, sendCallBack);
return new SendResult(true);
}
else
{
return new SendResult(true, "没有初始化生产者");
}
}
catch (Exception ex)
{
throw ex;
}
}
public ISendResult AsyncSend(Tkey key, TValue value,string topic)
{
try
{
if (producer != null)
{
var message = new Message<Tkey, TValue>
{
Value = value,
Key = key
};
var deliveryReport = producer.ProduceAsync(topic, message);
deliveryReport.ContinueWith(task =>
{
Console.WriteLine("Producer: " + producer.Name + "\r\nTopic: " + topic + "\r\nPartition: " + task.Result.Partition + "\r\nOffset: " + task.Result.Offset);
});
producer.Flush(TimeSpan.FromSeconds(10));
return new SendResult(true);
}
else
{
return new SendResult(true, "没有初始化生产者");
}
}
catch (Exception ex)
{
throw ex;
}
}
public ISendResult AsyncSend(Tkey key, TValue value, TopicPartition topicPartition)
{
try
{
if (producer != null)
{
var message = new Message<Tkey, TValue>
{
Value = value,
Key = key
};
var deliveryReport = producer.ProduceAsync(topicPartition, message);
deliveryReport.ContinueWith(task =>
{
Console.WriteLine("Producer: " + producer.Name + "\r\nTopic: " + topicPartition.Topic + "\r\nPartition: " + task.Result.Partition + "\r\nOffset: " + task.Result.Offset);
});
producer.Flush(TimeSpan.FromSeconds(10));
return new SendResult(true);
}
else
{
return new SendResult(true, "没有初始化生产者");
}
}
catch (Exception ex)
{
throw ex;
}
}新建一个SUPKafkaMessageCenter类这个类是对外开放的，我们利用这个类来管理生产者和消费者，看下代码非常简单。 public static class SUPKafkaMessageCenter<Tkey, TValue>
{
private static SUPLogger logger = null;
static SUPKafkaMessageCenter()
{
SUPLoggerManager.Configure();
logger = new SUPLogger("KafkaCenter");
}
/// <summary>
/// 创建生产者
/// </summary>
/// <param name="bootstrapServer"></param>
/// <param name="topicName"></param>
/// <returns></returns>
public static ISUPKafkaProducer<Tkey, TValue> CreateTopicProducer(string bootstrapServer)
{
if (string.IsNullOrEmpty(bootstrapServer))
{
return null;
}
var producer = new SUPKafkaTopicProducer<Tkey, TValue>(bootstrapServer, logger);
if (!producer.Init())
{
return null;
}
return producer;
}
/// <summary>
/// 创建消费者
/// </summary>
/// <param name="bootstrapServer"></param>
/// <param name="groupId"></param>
/// <returns></returns>
public static SUPKafkaTopicConsumer<Tkey, TValue> CreateTopicConsumer(string bootstrapServer, string groupId= "default-consumer-group")
{
if (string.IsNullOrEmpty(bootstrapServer))
{
return null;
}
var consumer = new SUPKafkaTopicConsumer<Tkey, TValue>(bootstrapServer, groupId,logger);
if (!consumer.Init())
{
return null;
}
return consumer;
}测试新建一个测试的控制台程序，调用代码如下
var consumer = SUPKafkaMessageCenter<string, string>.CreateTopicConsumer("localhost:9092");
//绑定接收信息，回调函数
consumer.onReceivedHandle += CallBack;
var topics = new List<string>();
topics.Add("kafka-default-topic");
topics.Add("test");
//订阅主题
consumer.Subscribe(topics, false);
ISUPKafkaProducer<string, string> kafkaCenter = SUPKafkaMessageCenter<string, string>.CreateTopicProducer("localhost:9092");
kafkaCenter.Send(i.ToString(), "", "kafka-default-topic",deliveryReport =>{...});除了上面写的这些方法，其实对于kafka还有很多功能，比如topic的增删改查，我把它认为是管理类的，这里就不贴代码了。 |
|
|
