Merge branch 'dev'

This commit is contained in:
Koen Kanters
2022-08-01 17:42:35 +02:00
20 changed files with 2065 additions and 1877 deletions
+3 -10
View File
@@ -85,24 +85,17 @@ jobs:
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Cache node-gyp
uses: actions/cache@v3
with:
key: ${{ matrix.os }}-${{ matrix.node }}-node-gyp
path: |
/home/runner/.cache/node-gyp
C:\Users\runneradmin\AppData\Local\node-gyp
/Users/runner/Library/Caches/node-gyp
- uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node }}
registry-url: https://registry.npmjs.org/
cache: 'npm'
- name: Install dependencies
run: npm ci
# --ignore-scripts prevents the serialport build which often fails on Windows
run: npm ci --ignore-scripts
- name: Lint
run: npm run eslint
- name: Build
run: npm run build
- name: Test
run: npm run test-with-coverage
run: npm run test-with-coverage
+1 -1
View File
@@ -11,4 +11,4 @@ jobs:
- name: 'Checkout repository'
uses: actions/checkout@v3
- name: 'Dependency review'
uses: actions/dependency-review-action@v1
uses: actions/dependency-review-action@v2
+3 -5
View File
@@ -157,7 +157,7 @@ class Controller {
if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
for (const entity of [...devices, ...this.zigbee.groups()]) {
if (this.state.exists(entity)) {
this.publishEntityState(entity, this.state.get(entity));
this.publishEntityState(entity, this.state.get(entity), 'publishCached');
}
}
}
@@ -265,10 +265,8 @@ class Controller {
extension.adjustMessageBeforePublish?.(entity, message);
}
// filter mqtt message attributes
if (entity.options.filtered_attributes) {
entity.options.filtered_attributes.forEach((a) => delete message[a]);
}
// Filter mqtt message attributes
utils.filterProperties(entity.options.filtered_attributes, message);
if (Object.entries(message).length) {
const output = settings.get().advanced.output;
+2 -1
View File
@@ -13,7 +13,8 @@ const legacyApi = settings.get().advanced.legacy_api;
const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
'hvacThermostat', 'msTemperatureMeasurement'];
'hvacThermostat', 'msIlluminanceMeasurement', 'msTemperatureMeasurement', 'msRelativeHumidity',
'msSoilMoisture', 'msCO2'];
// See zigbee-herdsman-converters
const defaultBindGroup = {type: 'group_number', ID: 901, name: 'default_bind_group'};
+1 -1
View File
@@ -101,7 +101,7 @@ export default class Groups extends Extension {
@bind async onStateChange(data: eventdata.StateChange): Promise<void> {
const reason = 'groupOptimistic';
if (data.reason === reason) {
if (data.reason === reason || data.reason === 'publishCached') {
return;
}
+8 -1
View File
@@ -302,6 +302,7 @@ export default class HomeAssistant extends Extension {
command_topic: true,
command_topic_prefix: endpoint,
command_topic_postfix: tempCalibration.property,
device_class: 'temperature',
entity_category: 'config',
icon: 'mdi:math-compass',
...(tempCalibration.unit && {unit_of_measurement: tempCalibration.unit}),
@@ -754,6 +755,12 @@ export default class HomeAssistant extends Extension {
},
};
if (lookup[firstExpose.name]?.device_class === 'temperature') {
discoveryEntry.discovery_payload.device_class == lookup[firstExpose.name]?.device_class;
} else {
delete discoveryEntry.discovery_payload.device_class;
}
if (firstExpose.value_min != null) discoveryEntry.discovery_payload.min = firstExpose.value_min;
if (firstExpose.value_max != null) discoveryEntry.discovery_payload.max = firstExpose.value_max;
@@ -1336,7 +1343,7 @@ export default class HomeAssistant extends Extension {
// Publish all device states.
for (const entity of [...this.zigbee.devices(false), ...this.zigbee.groups()]) {
if (this.state.exists(entity)) {
this.publishEntityState(entity, this.state.get(entity));
this.publishEntityState(entity, this.state.get(entity), 'publishCached');
}
}
+2 -1
View File
@@ -256,7 +256,8 @@ export default class Publish extends Extension {
}
// filter out attribute listed in filtered_optimistic
entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
utils.filterProperties(entitySettings.filtered_optimistic, msg);
addToToPublish(re, msg);
}
+2 -2
View File
@@ -145,9 +145,9 @@ export default class Receive extends Extension {
if (converted) {
payload = {...payload, ...converted};
}
} catch (error) {
// istanbul ignore next
} catch (error) /* istanbul ignore next */ {
logger.error(`Exception while calling fromZigbee converter: ${error.message}}`);
logger.debug(error.stack);
}
}
+31 -1
View File
@@ -10,6 +10,10 @@ export default class MQTT {
private connectionTimer: NodeJS.Timeout;
private client: mqtt.MqttClient;
private eventBus: EventBus;
private initialConnect = true;
private republishRetainedTimer: NodeJS.Timer;
private retainedMessages: {[s: string]: {payload: string, options: MQTTOptions,
skipLog: boolean, skipReceive: boolean, topic: string, base: string}} = {};
constructor(eventBus: EventBus) {
this.eventBus = eventBus;
@@ -94,8 +98,20 @@ export default class MQTT {
}, utils.seconds(10));
logger.info('Connected to MQTT server');
if (this.initialConnect) {
await this.publishStateOnline();
} else {
this.republishRetainedTimer = setTimeout(() => {
// Republish retained messages in case MQTT broker does not persist them.
// https://github.com/Koenkk/zigbee2mqtt/issues/9629
Object.values(this.retainedMessages).forEach((e) =>
this.publish(e.topic, e.payload, e.options, e.base, e.skipLog, e.skipReceive));
}, 2000);
}
this.initialConnect = false;
this.subscribe(`${settings.get().mqtt.base_topic}/#`);
await this.publishStateOnline();
}
async publishStateOnline(): Promise<void> {
@@ -121,6 +137,11 @@ export default class MQTT {
logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
this.eventBus.emitMQTTMessage({topic, message: message + ''});
}
if (this.republishRetainedTimer && topic == `${settings.get().mqtt.base_topic}/bridge/state`) {
clearTimeout(this.republishRetainedTimer);
this.republishRetainedTimer = null;
}
}
isConnected(): boolean {
@@ -137,6 +158,15 @@ export default class MQTT {
this.publishedTopics.add(topic);
}
if (options.retain) {
if (payload) {
this.retainedMessages[topic] =
{payload, options, skipReceive, skipLog, topic: topic.substring(base.length + 1), base};
} else {
delete this.retainedMessages[topic];
}
}
this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});
if (!this.isConnected()) {
+9 -11
View File
@@ -1,15 +1,16 @@
import logger from './util/logger';
import data from './util/data';
import * as settings from './util/settings';
import utils from './util/utils';
import fs from 'fs';
import objectAssignDeep from 'object-assign-deep';
const saveInterval = 1000 * 60 * 5; // 5 minutes
const dontCacheProperties = [
'^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
'^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
'^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
];
class State {
@@ -74,17 +75,14 @@ class State {
set(entity: Group | Device, update: KeyValue, reason: string=null): KeyValue {
const fromState = this.state[entity.ID] || {};
const toState = objectAssignDeep({}, fromState, update);
const result = {...toState};
const newCache = {...toState};
const entityDontCacheProperties = entity.options.filtered_cache || [];
for (const property of Object.keys(toState)) {
if (dontCacheProperties.find((p) => property.match(p))) {
delete toState[property];
}
}
utils.filterProperties(dontCacheProperties.concat(entityDontCacheProperties), newCache);
this.state[entity.ID] = toState;
this.state[entity.ID] = newCache;
this.eventBus.emitStateChange({entity, from: fromState, to: toState, reason, update});
return result;
return toState;
}
remove(ID: string | number): void {
+5 -3
View File
@@ -42,7 +42,7 @@ declare global {
// Types
interface MQTTResponse {data: KeyValue, status: 'error' | 'ok', error?: string, transaction?: string}
interface MQTTOptions {qos?: mqtt.QoS, retain?: boolean, properties?: {messageExpiryInterval: number}}
type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged';
type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged' | 'publishCached';
type PublishEntityState = (entity: Device | Group, payload: KeyValue,
stateChangeReason?: StateChangeReason) => Promise<void>;
type RecursivePartial<T> = {[P in keyof T]?: RecursivePartial<T[P]>;};
@@ -280,11 +280,12 @@ declare global {
retrieve_state?: boolean,
debounce?: number,
debounce_ignore?: string[],
filtered_attributes?: string[],
filtered_cache?: string[],
filtered_optimistic?: string[],
icon?: string,
homeassistant?: KeyValue,
legacy?: boolean,
filtered_attributes?: string[],
friendly_name: string,
description?: string,
qos?: 0 | 1 | 2,
@@ -295,10 +296,11 @@ declare global {
ID?: number,
optimistic?: boolean,
off_state?: 'all_members_off' | 'last_member_state'
filtered_attributes?: string[],
filtered_cache?: string[],
filtered_optimistic?: string[],
retrieve_state?: boolean,
homeassistant?: KeyValue,
filtered_attributes?: string[],
friendly_name: string,
description?: string,
qos?: 0 | 1 | 2,
+20 -11
View File
@@ -812,23 +812,32 @@
"description": "Publish optimistic state after set",
"default": true
},
"filtered_optimistic": {
"type": "array",
"items": {
"type": "string"
},
"examples": ["color_mode", "color_temp", "color"],
"title": "Filtered optimistic attributes",
"description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
},
"filtered_attributes": {
"type": "array",
"items": {
"type": "string"
},
"examples": ["temperature", "battery", "action"],
"examples": ["^temperature$", "^battery$", "^action$"],
"title": "Filtered publish attributes",
"description": "Filter attributes from publish payload."
"description": "Filter attributes with regex from published payload."
},
"filtered_cache": {
"type": "array",
"items": {
"type": "string"
},
"examples": ["^input_actions$"],
"title": "Filtered attributes from cache",
"description": "Filter attributes with regex from being added to the cache, this prevents the attribute from being in the published payload when the value didn't change."
},
"filtered_optimistic": {
"type": "array",
"items": {
"type": "string"
},
"examples": ["^color_(mode|temp)$", "color"],
"title": "Filtered optimistic attributes",
"description": "Filter attributes with regex from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
},
"icon": {
"type": "string",
+5 -1
View File
@@ -381,7 +381,11 @@ function applyEnvironmentVariables(settings: Partial<Settings>): void {
}, settings);
if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
setting[key] = JSON.parse(process.env[envVariableName]);
try {
setting[key] = JSON.parse(process.env[envVariableName]);
} catch (error) {
setting[key] = process.env[envVariableName];
}
} else if (type.indexOf('number') >= 0) {
/* eslint-disable-line */ // @ts-ignore
setting[key] = process.env[envVariableName] * 1;
+10 -1
View File
@@ -341,6 +341,15 @@ function publishLastSeen(data: eventdata.LastSeenChanged, settings: Settings, al
}
}
function filterProperties(filter: string[], data: KeyValue): void {
if (filter) {
for (const property of Object.keys(data)) {
if (filter.find((p) => property.match(`^${p}$`))) {
delete data[property];
}
}
}
}
export default {
endpointNames, capitalize, getZigbee2MQTTVersion, getDependencyVersion, formatDate, objectHasProperties,
@@ -348,5 +357,5 @@ export default {
getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
sanitizeImageParameter, isAvailabilityEnabledForEntity, publishLastSeen, availabilityPayload,
getAllFiles,
getAllFiles, filterProperties,
};
+1875 -1805
View File
File diff suppressed because it is too large Load Diff
+20 -20
View File
@@ -1,6 +1,6 @@
{
"name": "zigbee2mqtt",
"version": "1.26.0",
"version": "1.26.0-dev",
"description": "Zigbee to MQTT bridge using Zigbee-herdsman",
"main": "index.js",
"repository": {
@@ -38,7 +38,7 @@
"ajv": "^8.11.0",
"bind-decorator": "^1.0.11",
"connect-gzip-static": "2.1.1",
"core-js": "^3.23.3",
"core-js": "^3.24.1",
"debounce": "^1.2.1",
"deep-object-diff": "^1.1.7",
"fast-deep-equal": "^3.1.3",
@@ -49,41 +49,41 @@
"json-stable-stringify-without-jsonify": "^1.0.1",
"jszip": "^3.10.0",
"mkdir-recursive": "^0.4.0",
"moment": "^2.29.3",
"moment": "^2.29.4",
"mqtt": "4.3.7",
"object-assign-deep": "^0.4.0",
"rimraf": "^3.0.2",
"semver": "^7.3.7",
"source-map-support": "^0.5.21",
"uri-js": "^4.4.1",
"winston": "^3.8.0",
"winston-syslog": "^2.5.0",
"winston": "^3.8.1",
"winston-syslog": "^2.6.0",
"winston-transport": "^4.5.0",
"ws": "^8.8.0",
"zigbee-herdsman": "0.14.40",
"zigbee-herdsman-converters": "14.0.559",
"zigbee2mqtt-frontend": "0.6.103"
"ws": "^8.8.1",
"zigbee-herdsman": "0.14.46",
"zigbee-herdsman-converters": "14.0.583",
"zigbee2mqtt-frontend": "0.6.107"
},
"devDependencies": {
"@babel/core": "^7.18.5",
"@babel/plugin-proposal-decorators": "^7.18.2",
"@babel/preset-env": "^7.18.2",
"@babel/preset-typescript": "^7.17.12",
"@babel/core": "^7.18.9",
"@babel/plugin-proposal-decorators": "^7.18.9",
"@babel/preset-env": "^7.18.9",
"@babel/preset-typescript": "^7.18.6",
"@types/debounce": "^1.2.1",
"@types/finalhandler": "^1.1.1",
"@types/humanize-duration": "^3.27.1",
"@types/jest": "^28.1.3",
"@types/jest": "^28.1.6",
"@types/js-yaml": "^4.0.5",
"@types/object-assign-deep": "^0.4.0",
"@types/rimraf": "^3.0.2",
"@types/ws": "^8.5.3",
"@typescript-eslint/eslint-plugin": "^5.29.0",
"@typescript-eslint/parser": "^5.29.0",
"babel-jest": "^28.1.1",
"eslint": "^8.18.0",
"@typescript-eslint/eslint-plugin": "^5.31.0",
"@typescript-eslint/parser": "^5.31.0",
"babel-jest": "^28.1.3",
"eslint": "^8.20.0",
"eslint-config-google": "^0.14.0",
"eslint-plugin-jest": "^26.5.3",
"jest": "^28.1.1",
"eslint-plugin-jest": "^26.7.0",
"jest": "^28.1.3",
"tmp": "^0.2.1",
"typescript": "^4.7.4"
},
+60
View File
@@ -490,6 +490,46 @@ describe('Controller', () => {
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
});
it('Publish entity state attribute_json output filtered cache', async () => {
await controller.start();
settings.set(['advanced', 'output'], 'attribute_and_json');
settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_cache'], ['linkquality']);
MQTT.publish.mockClear();
const device = controller.zigbee.resolveEntity('bulb');
expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
await flushPromises();
expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
expect(MQTT.publish).toHaveBeenCalledTimes(5);
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
});
it('Publish entity state attribute_json output filtered cache (device_options)', async () => {
await controller.start();
settings.set(['advanced', 'output'], 'attribute_and_json');
settings.set(['device_options', 'filtered_cache'], ['linkquality']);
MQTT.publish.mockClear();
const device = controller.zigbee.resolveEntity('bulb');
expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":50,"color_temp":370,"linkquality":99,"state":"ON"});
await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 87});
await flushPromises();
expect(controller.state.state[device.ieeeAddr]).toStrictEqual({"brightness":200,"color_temp":370,"state":"ON"});
expect(MQTT.publish).toHaveBeenCalledTimes(5);
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "87", {"qos": 0, "retain": true}, expect.any(Function));
expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":87}), {"qos": 0, "retain": true}, expect.any(Function));
});
it('Publish entity state with device information', async () => {
await controller.start();
settings.set(['mqtt', 'include_device_information'], true);
@@ -605,6 +645,26 @@ describe('Controller', () => {
expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
});
it('Should republish retained messages on MQTT reconnect', async () => {
await controller.start();
MQTT.publish.mockClear();
MQTT.events['connect']();
await flushPromises();
jest.runOnlyPendingTimers();
expect(MQTT.publish).toHaveBeenCalledTimes(12);
expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
});
it('Should not republish retained messages on MQTT reconnect when retained message are sent', async () => {
await controller.start();
MQTT.publish.mockClear();
MQTT.events['connect']();
await flushPromises();
await MQTT.events.message('zigbee2mqtt/bridge/state', 'online');
jest.runOnlyPendingTimers();
expect(MQTT.publish).toHaveBeenCalledTimes(0);
});
it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
settings.set(['mqtt', 'force_disable_retain'], true);
await controller.mqtt.connect()
+2 -2
View File
@@ -250,7 +250,7 @@ describe('Receive', () => {
await flushPromises();
expect(MQTT.publish).toHaveBeenCalledTimes(1);
expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 46, voltage: 3010});
expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
});
@@ -262,7 +262,7 @@ describe('Receive', () => {
await flushPromises();
expect(MQTT.publish).toHaveBeenCalledTimes(1);
expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'device_temperature': 19, 'power_outage_count': 34});
expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 56, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'device_temperature': 19, 'power_outage_count': 34});
expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
});
+2
View File
@@ -70,6 +70,7 @@ describe('Settings', () => {
process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_OUTPUT'] = 'csvtest';
process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_NETWORK_KEY'] = 'GENERATE';
write(configurationFile, {});
const s = settings.get();
@@ -81,6 +82,7 @@ describe('Settings', () => {
expected.advanced.output = 'csvtest';
expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
expected.mqtt.base_topic = 'testtopic';
expected.advanced.network_key = 'GENERATE';
expect(s).toStrictEqual(expected);
});
+4
View File
@@ -27,7 +27,11 @@ const clusters = {
'lightingColorCtrl': 768,
'closuresWindowCovering': 258,
'hvacThermostat': 513,
'msIlluminanceMeasurement': 1024,
'msTemperatureMeasurement': 1026,
'msRelativeHumidity': 1029,
'msSoilMoisture': 1032,
'msCO2': 1037
}
class Endpoint {