Automatically disable a button once there's no associated element left

I have a list of several elements, and some controlling buttons that can hide/show these elements. Some buttons have control over just one element, while others have multiple elements.

What does my code do:

• Button01 hides/shows ElementX and ElementY,
• Button02 hides/shows only ElementY.
• Once Button01 is clicked, both of the elements are hidden, and clicking Button02 doesn't change anything until Button01 is clicked again.

What I want to do:

• Once Button01 hides ElementX and ElementY, Button02 must also be grayed out automatically because its associated element is gone.
• And then, clicking Button02 should bring ElementY back and enable Button01 too since one of the associated elements of Button01 is back.

for (let button of document.querySelectorAll(".filterbutton")) {
    button.addEventListener("click", filter);
}

let filters = new Set;

function toggleDisplay(selector, display) {
    let elems = document.querySelectorAll(selector);
    for (let elem of elems) {
        elem.style.display = display;
    }
}

function filter() {
    let filterSelector = this.dataset.filter;
    let show = filters.delete(filterSelector);
    this.style.color = show ? "" : "rgb(200,200,200)";
    if (!show) {
        filters.add(filterSelector); // toggle this filter
    } else {
        toggleDisplay(filterSelector, "");
    }
    if (filters.size) {
        toggleDisplay([...filters].join(","), "none");
    }
}

<div class="filterbutton" data-filter=".filter01">Filter01</div>
<div class="filterbutton" data-filter=".filter02">Filter02</div>

<div class="filter01">ElementX</div>
<div class="filter01 filter02">ElementY</div>