diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c
index 9e8b8aab049d..62485a57dbfc 100644
--- a/net/dsa/dsa2.c
+++ b/net/dsa/dsa2.c
@@ -260,8 +260,6 @@ static void dsa_cpu_port_unapply(struct dsa_port *port)
 {
 	devlink_port_unregister(&port->devlink_port);
 	dsa_cpu_dsa_destroy(port);
-	port->ds->cpu_port_mask &= ~BIT(port->index);
-
 }
 
 static int dsa_user_port_apply(struct dsa_port *port)
@@ -300,7 +298,6 @@ static void dsa_user_port_unapply(struct dsa_port *port)
 	if (port->slave) {
 		dsa_slave_destroy(port->slave);
 		port->slave = NULL;
-		port->ds->enabled_port_mask &= ~(1 << port->index);
 	}
 }
 
@@ -512,7 +509,6 @@ static int dsa_cpu_parse(struct dsa_port *port, u32 index,
 	tag_ops = dsa_resolve_tag_protocol(tag_protocol);
 	if (IS_ERR(tag_ops)) {
 		dev_warn(ds->dev, "No tagger for this switch\n");
-		ds->cpu_port_mask &= ~BIT(index);
 		return PTR_ERR(tag_ops);
 	}
 
diff --git a/net/dsa/legacy.c b/net/dsa/legacy.c
index b6c88fd33d4f..93c1c43bcc58 100644
--- a/net/dsa/legacy.c
+++ b/net/dsa/legacy.c
@@ -272,10 +272,6 @@ static void dsa_switch_destroy(struct dsa_switch *ds)
 		if (!(dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)))
 			continue;
 		dsa_cpu_dsa_destroy(&ds->ports[port]);
-
-		/* Clearing a bit which is not set does no harm */
-		ds->cpu_port_mask |= ~(1 << port);
-		ds->dsa_port_mask |= ~(1 << port);
 	}
 
 	if (ds->slave_mii_bus && ds->ops->phy_read)